<!DOCTYPE html>
<!-- saved from url=(0116)https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec -->
<html xmlns:cc="http://creativecommons.org/ns#"><head prefix="og: http://ogp.me/ns# fb: http://ogp.me/ns/fb# medium-com: http://ogp.me/ns/fb/medium-com#"><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=contain"><title>Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT</title><link rel="canonical" href="https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec"><meta name="title" content="Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT"><meta name="referrer" content="always"><meta name="description" content="Since the advent of word2vec, neural word embeddings have become a go to method for encapsulating distributional semantics in text applications. This series will review the strengths and weaknesses…"><meta name="theme-color" content="#000000"><meta property="og:title" content="Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT"><meta property="twitter:title" content="Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT"><meta property="og:url" content="https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec"><meta property="og:image" content="https://cdn-images-1.medium.com/max/1200/0*K8eg3bUVu4AG-4FB"><meta property="fb:app_id" content="542599432471018"><meta property="og:description" content="A primer in the neural nlp model archticture and word representation."><meta name="twitter:description" content="A primer in the neural nlp model archticture and word representation."><meta name="twitter:image:src" content="https://cdn-images-1.medium.com/max/1200/0*K8eg3bUVu4AG-4FB"><link rel="author" href="https://towardsdatascience.com/@aribornstein"><meta name="author" content="Aaron (Ari) Bornstein"><meta property="og:type" content="article"><meta name="twitter:card" content="summary"><meta property="article:publisher" content="https://www.facebook.com/towardsdatascience"><meta property="article:author" content="Aaron (Ari) Bornstein"><meta name="robots" content="index, follow"><meta property="article:published_time" content="2018-10-17T16:45:18.913Z"><meta name="twitter:site" content="@TDataScience"><meta property="og:site_name" content="Towards Data Science"><meta name="twitter:label1" value="Reading time"><meta name="twitter:data1" value="10 min read"><meta name="twitter:app:name:iphone" content="Medium"><meta name="twitter:app:id:iphone" content="828256236"><meta name="twitter:app:url:iphone" content="medium://p/4ebd4711d0ec"><meta property="al:ios:app_name" content="Medium"><meta property="al:ios:app_store_id" content="828256236"><meta property="al:android:package" content="com.medium.reader"><meta property="al:android:app_name" content="Medium"><meta property="al:ios:url" content="medium://p/4ebd4711d0ec"><meta property="al:android:url" content="medium://p/4ebd4711d0ec"><meta property="al:web:url" content="https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec"><link rel="search" type="application/opensearchdescription+xml" title="Medium" href="https://towardsdatascience.com/osd.xml"><link rel="alternate" href="android-app://com.medium.reader/https/medium.com/p/4ebd4711d0ec"><script async="" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/branch-latest.min.js"></script><script type="application/ld+json">{"@context":"http://schema.org","@type":"NewsArticle","image":{"@type":"ImageObject","width":1010,"height":376,"url":"https://cdn-images-1.medium.com/max/1010/1*5EUO1kUYBthpOCPzRj_l2g.png"},"url":"https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec","dateCreated":"2018-10-17T16:45:18.913Z","datePublished":"2018-10-17T16:45:18.913Z","dateModified":"2018-12-17T12:26:39.327Z","headline":"Beyond Word Embeddings Part 2- Word Vectors & NLP Modeling from BoW to BERT","name":"Beyond Word Embeddings Part 2- Word Vectors & NLP Modeling from BoW to BERT","articleId":"4ebd4711d0ec","thumbnailUrl":"https://cdn-images-1.medium.com/max/1010/1*5EUO1kUYBthpOCPzRj_l2g.png","keywords":["Tag:Machine Learning","Tag:NLP","Tag:AI","Tag:Deep Learning","Tag:Data Science","Topic:Machine Learning","Topic:Data Science","Publication:towards-data-science","LockedPostSource:0","Elevated:false","LayerCake:3"],"author":{"@type":"Person","name":"Aaron (Ari) Bornstein","url":"https://towardsdatascience.com/@aribornstein"},"creator":["Aaron (Ari) Bornstein"],"publisher":{"@type":"Organization","name":"Towards Data Science","url":"https://towardsdatascience.com","logo":{"@type":"ImageObject","width":161,"height":60,"url":"https://cdn-images-1.medium.com/max/161/1*5EUO1kUYBthpOCPzRj_l2g.png"}},"mainEntityOfPage":"https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec"}</script><meta name="parsely-link" content="https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec"><link rel="stylesheet" type="text/css" class="js-glyph-" id="glyph-8" href="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/m2.css"><link rel="stylesheet" href="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/main-branding-base.Oq6YIB3xxaj4PGdhFuOAaA.css"><script>!function(n,e){var t,o,i,c=[],f={passive:!0,capture:!0},r=new Date,a="pointerup",u="pointercancel";function p(n,c){t||(t=c,o=n,i=new Date,w(e),s())}function s(){o>=0&&o<i-r&&(c.forEach(function(n){n(o,t)}),c=[])}function l(t){if(t.cancelable){var o=(t.timeStamp>1e12?new Date:performance.now())-t.timeStamp;"pointerdown"==t.type?function(t,o){function i(){p(t,o),r()}function c(){r()}function r(){e(a,i,f),e(u,c,f)}n(a,i,f),n(u,c,f)}(o,t):p(o,t)}}function w(n){["click","mousedown","keydown","touchstart","pointerdown"].forEach(function(e){n(e,l,f)})}w(n),self.perfMetrics=self.perfMetrics||{},self.perfMetrics.onFirstInputDelay=function(n){c.push(n),s()}}(addEventListener,removeEventListener);</script><script>if (window.top !== window.self) window.top.location = window.self.location.href;var OB_startTime = new Date().getTime(); var OB_loadErrors = []; function _onerror(e) { OB_loadErrors.push(e) }; if (document.addEventListener) document.addEventListener("error", _onerror, true); else if (document.attachEvent) document.attachEvent("onerror", _onerror); function _asyncScript(u) {var d = document, f = d.getElementsByTagName("script")[0], s = d.createElement("script"); s.type = "text/javascript"; s.async = true; s.src = u; f.parentNode.insertBefore(s, f);}function _asyncStyles(u) {var d = document, f = d.getElementsByTagName("script")[0], s = d.createElement("link"); s.rel = "stylesheet"; s.href = u; f.parentNode.insertBefore(s, f); return s}(new Image()).src = "/_/stat?event=pixel.load&origin=" + encodeURIComponent(location.origin);</script><script>window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date; ga("create", "UA-24232453-2", "auto", {"allowLinker": true, "legacyCookieDomain": window.location.hostname}); ga("send", "pageview");ga("create", "UA-19707169-24", "auto", 'tracker0'); ga("tracker0.send", "pageview");</script><script async="" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/analytics.js"></script><!--[if lt IE 9]><script charset="UTF-8" src="https://cdn-static-1.medium.com/_/fp/js/shiv.RI2ePTZ5gFmMgLzG5bEVAA.js"></script><![endif]--><link rel="icon" href="https://cdn-images-1.medium.com/fit/c/128/128/1*F0LADxTtsKOgmPa-_7iUEQ.jpeg" class="js-favicon"><link rel="apple-touch-icon" sizes="152x152" href="https://cdn-images-1.medium.com/fit/c/304/304/1*F0LADxTtsKOgmPa-_7iUEQ.jpeg"><link rel="apple-touch-icon" sizes="120x120" href="https://cdn-images-1.medium.com/fit/c/240/240/1*F0LADxTtsKOgmPa-_7iUEQ.jpeg"><link rel="apple-touch-icon" sizes="76x76" href="https://cdn-images-1.medium.com/fit/c/152/152/1*F0LADxTtsKOgmPa-_7iUEQ.jpeg"><link rel="apple-touch-icon" sizes="60x60" href="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_F0LADxTtsKOgmPa-_7iUEQ.jpeg"><link rel="mask-icon" href="https://cdn-static-1.medium.com/_/fp/icons/monogram-mask.KPLCSFEZviQN0jQ7veN2RQ.svg" color="#171717"></head><body itemscope="" class="postShowScreen browser-chrome os-mac is-withMagicUnderlines v-glyph v-glyph--m2 is-js" data-action-scope="_actionscope_0"><script>document.body.className = document.body.className.replace(/(^|\s)is-noJs(\s|$)/, "$1is-js$2")</script><div class="site-main surface-container" id="container"><div class="butterBar butterBar--error" data-action-scope="_actionscope_1"></div><div class="surface" id="_obv.shell._surface_1561519666314" style="display: block; visibility: visible;"><div class="screenContent surface-content is-supplementalPostContentLoaded" data-used="true" data-action-scope="_actionscope_2"><canvas class="canvas-renderer" width="1263" height="649"></canvas><div class="container u-maxWidth740 u-xs-margin0 notesPositionContainer js-notesPositionContainer"><div class="notesMarkers" data-action-scope="_actionscope_4"><div class="paragraphControls js-paragraphControl js-paragraphControl-7adf u-noUserSelect is-visible" style="top: 2112px;"><div class="notesMarker u-noUserSelect" data-action="select-anchor" data-action-value="7adf"><span class="svgIcon svgIcon--asteriskFilled svgIcon--19px"><svg class="svgIcon-use" width="19" height="19"><path d="M14.78 8.07a8.681 8.681 0 0 0-.427-1.383.478.478 0 0 0-.584-.27l-3.12.77V4.034c0-.247-.19-.48-.43-.5a7.23 7.23 0 0 0-1.38 0c-.24.02-.43.253-.43.5V7.19L5.3 6.415a.48.48 0 0 0-.583.27c-.18.448-.324.91-.426 1.383-.05.24.1.5.32.58l3.06.754-1.98 2.956c-.14.196-.13.502.04.67.34.332.7.632 1.09.896.2.136.49.077.63-.117l2.09-3.114 2.09 3.112c.15.193.43.252.63.116.39-.26.75-.56 1.09-.89.17-.17.19-.47.04-.67L11.4 9.41l3.06-.76a.517.517 0 0 0 .32-.58" fill-rule="evenodd"></path></svg></span></div><span class="paragraphControls-itemText"><button class="button button--chromeless" data-action="select-anchor" data-action-value="7adf"></button></span></div><div class="paragraphControls js-paragraphControl js-paragraphControl-3d70 u-noUserSelect is-visible" style="top: 4148px;"><span class="paragraphControls-itemText"><button class="button button--chromeless" data-action="select-anchor" data-action-value="3d70">Top highlight</button></span></div></div></div><div class="metabar u-clearfix u-boxShadow4px12pxBlackLighter u-textColorTransparentWhiteDarker js-metabar is-withBottomSection is-hiddenWhenMinimized is-maximized"><div class="branch-journeys-top"></div><div class="js-metabarMiddle metabar-inner u-marginAuto u-maxWidth1032 u-flexCenter u-justifyContentSpaceBetween u-height65 u-xs-height56 u-paddingHorizontal20"><div class="metabar-block u-flex1 u-flexCenter"><div class="js-metabarLogoLeft"><a href="https://medium.com/" data-log-event="home" class="siteNav-logo u-fillTransparentBlackDarker u-flex0 u-flexCenter u-paddingTop0"><span class="svgIcon svgIcon--logoMonogram svgIcon--45px"><svg class="svgIcon-use" width="45" height="45"><path d="M5 40V5h35v35H5zm8.56-12.627c0 .555-.027.687-.318 1.03l-2.457 2.985v.396h6.974v-.396l-2.456-2.985c-.291-.343-.344-.502-.344-1.03V18.42l6.127 13.364h.714l5.256-13.364v10.644c0 .29 0 .342-.185.528l-1.848 1.796v.396h9.19v-.396l-1.822-1.796c-.184-.186-.21-.238-.21-.528V15.937c0-.291.026-.344.21-.528l1.823-1.797v-.396h-6.471l-4.622 11.542-5.203-11.542h-6.79v.396l2.14 2.64c.239.292.291.37.291.768v10.353z"></path></svg></span><span class="u-textScreenReader">Homepage</span></a></div></div><div class="metabar-block u-flex0 u-flexCenter"><div class="u-flexCenter u-height65 u-xs-height56"><div class="buttonSet buttonSet--wide u-lineHeightInherit"><label class="button button--small button--chromeless button--withIcon button--withSvgIcon inputGroup u-sm-hide metabar-predictiveSearch u-baseColor--buttonNormal u-baseColor--placeholderNormal" title="Search"><span class="svgIcon svgIcon--search svgIcon--25px u-baseColor--iconLight"><svg class="svgIcon-use" width="25" height="25"><path d="M20.067 18.933l-4.157-4.157a6 6 0 1 0-.884.884l4.157 4.157a.624.624 0 1 0 .884-.884zM6.5 11c0-2.62 2.13-4.75 4.75-4.75S16 8.38 16 11s-2.13 4.75-4.75 4.75S6.5 13.62 6.5 11z"></path></svg></span><input class="js-predictiveSearchInput textInput textInput--rounded textInput--darkText u-baseColor--textNormal textInput--transparent" type="search" placeholder="Search" required="true" data-collection-id="7f60cf5620c9"></label><a class="button button--small button--chromeless u-sm-show is-inSiteNavBar u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--chromeless u-xs-top1" href="https://towardsdatascience.com/search" title="Search" aria-label="Search"><span class="button-defaultState"><span class="svgIcon svgIcon--search svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M20.067 18.933l-4.157-4.157a6 6 0 1 0-.884.884l4.157 4.157a.624.624 0 1 0 .884-.884zM6.5 11c0-2.62 2.13-4.75 4.75-4.75S16 8.38 16 11s-2.13 4.75-4.75 4.75S6.5 13.62 6.5 11z"></path></svg></span></span></a><button class="button button--small button--chromeless is-inSiteNavBar u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--activity js-notificationsButton u-marginRight16 u-xs-marginRight10 u-lineHeight0 u-size25x25" title="Notifications" aria-label="Notifications" data-action="open-notifications"><span class="svgIcon svgIcon--bell svgIcon--25px"><svg class="svgIcon-use" width="25" height="25" viewBox="-293 409 25 25"><path d="M-273.327 423.67l-1.673-1.52v-3.646a5.5 5.5 0 0 0-6.04-5.474c-2.86.273-4.96 2.838-4.96 5.71v3.41l-1.68 1.553c-.204.19-.32.456-.32.734V427a1 1 0 0 0 1 1h3.49a3.079 3.079 0 0 0 3.01 2.45 3.08 3.08 0 0 0 3.01-2.45h3.49a1 1 0 0 0 1-1v-2.59c0-.28-.12-.55-.327-.74zm-7.173 5.63c-.842 0-1.55-.546-1.812-1.3h3.624a1.92 1.92 0 0 1-1.812 1.3zm6.35-2.45h-12.7v-2.347l1.63-1.51c.236-.216.37-.522.37-.843v-3.41c0-2.35 1.72-4.356 3.92-4.565a4.353 4.353 0 0 1 4.78 4.33v3.645c0 .324.137.633.376.85l1.624 1.477v2.373z"></path></svg></span></button><button class="button button--chromeless u-baseColor--buttonNormal is-inSiteNavBar js-userActions" aria-haspopup="true" data-action="open-userActions"><div class="avatar"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_Mpgm7EXaTEICpUOi.jpg" class="avatar-image avatar-image--icon" alt="Cheng-Jun Wang"></div></button></div></div></div></div><div class="u-tintBgColor u-tintSpectrum "><div class="metabar-inner u-marginAuto u-maxWidth1032 u-paddingHorizontal20 js-metabarBottom"><nav role="navigation" class="metabar-block metabar-block--below u-flexCenter u-overflowHidden u-height54"><div class="u-flexCenter u-overflowHidden"><div class="u-marginRight40"><a href="https://towardsdatascience.com/?source=logo-3751a3493996---7f60cf5620c9" class="u-flexCenter js-collectionLogoOrName"><img height="36" width="97" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_5EUO1kUYBthpOCPzRj_l2g.png" alt="Towards Data Science"></a></div><div class="u-overflowHidden u-xs-hide"><ul class="u-textAlignLeft u-noWrap u-overflowX u-height80 u-marginTop40 js-collectionNavItems"><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/data-science/home">Data Science</a></li><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/machine-learning/home">Machine Learning</a></li><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/programming/home">Programming</a></li><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/data-visualization/home">Visualization</a></li><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/artificial-intelligence/home">AI</a></li><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/data-journalism/home">Journalism</a></li><li class="metabar-navItem js-collectionNavItem u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darken u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/editors-picks/home">Picks</a></li><span class="u-borderLeft1 u-baseColor--borderLight"></span><li class="metabar-navItem js-collectionNavItem is-external u-inlineBlock u-fontSize13 u-textUppercase u-letterSpacing1px u-textColorNormal u-xs-paddingRight12 u-xs-marginRight0 u-xs-paddingTop10"><a class="link link--darkenOnHover u-accentColor--textDarken link--noUnderline u-baseColor--link js-navItemLink" href="https://towardsdatascience.com/contribute/home" rel="nofollow noopener" target="_blank">Contribute</a></li></ul></div></div></nav></div></div></div><div class="metabar metabar--spacer js-metabarSpacer u-tintBgColor  u-height119 u-xs-height110"></div><main role="main"><article class=" u-minHeight100vhOffset65 u-overflowHidden postArticle postArticle--full is-withAccentColors" lang="en"><div class="postArticle-content js-postField js-notesSource js-trackPostScrolls" data-post-id="4ebd4711d0ec" data-source="post_page" data-collection-id="7f60cf5620c9" data-tracking-context="postPage" data-scroll="native"><section name="596c" class="section section--body section--first"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><h1 name="1d85" id="1d85" class="graf graf--h3 graf--leading graf--title">Beyond Word Embeddings Part&nbsp;2</h1><div class="uiScale uiScale-ui--regular uiScale-caption--regular u-flexCenter u-marginVertical24 u-fontSize15 js-postMetaLockup"><div class="u-flex0"><a class="link u-baseColor--link avatar" href="https://towardsdatascience.com/@aribornstein?source=post_header_lockup" data-action="show-user-card" data-action-source="post_header_lockup" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" data-collection-slug="towards-data-science" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_9Y0zWLwh1nYuBZMetDnC7w.jpeg" class="avatar-image u-size50x50" alt="Go to the profile of Aaron (Ari) Bornstein"></a></div><div class="u-flex1 u-paddingLeft15 u-overflowHidden"><div class="u-paddingBottom3"><a class="ds-link ds-link--styleSubtle ui-captionStrong u-inlineBlock link link--darken link--darker" href="https://towardsdatascience.com/@aribornstein" data-action="show-user-card" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" data-collection-slug="towards-data-science" dir="auto">Aaron (Ari) Bornstein</a><span class="followState js-followState" data-user-id="b3c7769e3e2f"><button class="button button--smallest u-noUserSelect button--withChrome u-baseColor--buttonNormal button--withHover button--unblock js-unblockButton u-marginLeft10 u-xs-hide" data-action="toggle-block-user" data-action-value="b3c7769e3e2f" data-action-source="post_header_lockup"><span class="button-label  button-defaultState">Blocked</span><span class="button-label button-hoverState">Unblock</span></button><button class="button button--primary button--smallest button--dark u-noUserSelect button--withChrome u-accentColor--buttonDark button--follow js-followButton u-marginLeft10 u-xs-hide" data-action="toggle-subscribe-user" data-action-value="b3c7769e3e2f" data-action-source="post_header_lockup-b3c7769e3e2f-------------------------follow_byline" data-subscribe-source="post_header_lockup" data-follow-context-entity-id="4ebd4711d0ec"><span class="button-label  button-defaultState js-buttonLabel">Follow</span><span class="button-label button-activeState">Following</span></button></span></div><div class="ui-caption u-noWrapWithEllipsis js-testPostMetaInlineSupplemental"><time datetime="2018-10-17T16:45:18.913Z">Oct 18, 2018</time><span class="middotDivider u-fontSize12"></span><span class="readingTime" title="10 min read"></span></div></div></div><p name="1f51" id="1f51" class="graf graf--p graf-after--h3">Word Vectors and NLP Modeling from BoW to BERT</p><figure name="c838" id="c838" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder"><img class="graf-image" data-image-id="0*K8eg3bUVu4AG-4FB" data-is-featured="true" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_K8eg3bUVu4AG-4FB"></div></figure><h3 name="081b" id="081b" class="graf graf--h3 graf-after--figure">TL;DR</h3><p name="5bd9" id="5bd9" class="graf graf--p graf-after--h3 graf--trailing">Since the advent of <a href="https://en.wikipedia.org/wiki/Word2vec" data-href="https://en.wikipedia.org/wiki/Word2vec" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">word2vec</a>, neural word embeddings have become a go to method for encapsulating distributional semantics in text applications. This series will review the strengths and weaknesses of using pre-trained word embeddings and demonstrate how to incorporate more complex semantic representation schemes such as Semantic Role Labeling, Abstract Meaning Representation and Semantic Dependency Parsing into your applications.</p></div></div></section><section name="2fc6" class="section section--body"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><h4 name="746d" id="746d" class="graf graf--h4 graf--leading">Introduction</h4><p name="ecc8" id="ecc8" class="graf graf--p graf-after--h4">The <a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" class="markup--anchor markup--p-anchor" target="_blank">last post in this series</a> reviewed some of the recent milestones in neural natural language processing. In this post we will review some of the advancements in text representation.</p><p name="7db2" id="7db2" class="graf graf--p graf-after--p graf--trailing">Computers are unable to understand the concepts of words. In order to process natural language a mechanism for representing text is required. The standard mechanism for text representation are <strong class="markup--strong markup--p-strong">word vectors</strong> where<em class="markup--em markup--p-em"> words or phrases from a given language vocabulary are mapped to </em><a href="https://en.wikipedia.org/wiki/Array_data_structure" data-href="https://en.wikipedia.org/wiki/Array_data_structure" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><em class="markup--em markup--p-em">vectors</em></a><em class="markup--em markup--p-em"> of </em><a href="https://en.wikipedia.org/wiki/Real_number" data-href="https://en.wikipedia.org/wiki/Real_number" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><em class="markup--em markup--p-em">real numbers</em></a><em class="markup--em markup--p-em">.</em></p></div></div></section><section name="b50e" class="section section--body"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><h3 name="9cfa" id="9cfa" class="graf graf--h3 graf--leading"><strong class="markup--strong markup--h3-strong">Traditional Word&nbsp;Vectors</strong></h3><p name="7ea6" id="7ea6" class="graf graf--p graf-after--h3">Before diving directly into Word2Vec it’s worth while to do a brief overview of some of the traditional methods that pre-date neural embeddings.</p><p name="a41d" id="a41d" class="graf graf--p graf-after--p"><strong class="markup--strong markup--p-strong">Bag of Words</strong> or BoW vector representations are the most common used traditional vector representation. Each word or n-gram is linked to a vector index and marked as 0 or 1 depending on whether it occurs in a given document.</p><figure name="306e" id="306e" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 151px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 21.5%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="1*dbx2P57arL8jrO-0QPrEKw.png" data-width="757" data-height="163" data-action="zoom" data-action-value="1*dbx2P57arL8jrO-0QPrEKw.png" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_dbx2P57arL8jrO-0QPrEKw.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="15"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/1*dbx2P57arL8jrO-0QPrEKw.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_dbx2P57arL8jrO-0QPrEKw(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/1*dbx2P57arL8jrO-0QPrEKw.png"></noscript></div></div><figcaption class="imageCaption">An example of a one hot bag of words representation for documents with one&nbsp;word.</figcaption></figure><p name="7adf" id="7adf" class="graf graf--p graf-after--figure">BoW representations are often used in methods of document classification where the frequency of each word, bi-word or tri-word is a useful feature for training classifiers. One challenge with bag of word representations is that they don’t encode any information with regards to the meaning of a given word.</p><p name="76f0" id="76f0" class="graf graf--p graf-after--p">In BoW word occurrences are evenly weighted independently of how frequently or what context they occur. However in most NLP tasks some words are more relevant than others.</p><p name="bcea" id="bcea" class="graf graf--p graf-after--p"><strong class="markup--strong markup--p-strong">TF-IDF</strong>, short for <a href="https://en.wikipedia.org/wiki/Tf%E2%80%93idf" data-href="https://en.wikipedia.org/wiki/Tf%E2%80%93idf" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><strong class="markup--strong markup--p-strong">term frequency–inverse document frequency</strong></a>, is a numerical statistic that is intended to reflect how important a word or n-gram is to a <a href="https://en.wikipedia.org/wiki/Document" data-href="https://en.wikipedia.org/wiki/Document" class="markup--anchor markup--p-anchor" title="Document" rel="noopener" target="_blank">document</a> in a collection or <a href="https://en.wikipedia.org/wiki/Text_corpus" data-href="https://en.wikipedia.org/wiki/Text_corpus" class="markup--anchor markup--p-anchor" title="Text corpus" rel="noopener" target="_blank">corpus</a>. They provide some weighting to a given word based on the context it occurs.The tf–idf value increases <a href="https://en.wikipedia.org/wiki/Proportionality_%28mathematics%29" data-href="https://en.wikipedia.org/wiki/Proportionality_%28mathematics%29" class="markup--anchor markup--p-anchor" title="Proportionality (mathematics)" rel="noopener" target="_blank">proportionally</a> to the number of times a word appears in a document and is offset by the number of documents in the corpus that contain the word, which helps to adjust for the fact that some words appear more frequently than others.</p><figure name="70a6" id="70a6" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 319px; max-height: 200px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 62.7%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*B67OTicNvuJaElW6.png" data-width="319" data-height="200" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_B67OTicNvuJaElW6.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="46"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*B67OTicNvuJaElW6.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_B67OTicNvuJaElW6(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*B67OTicNvuJaElW6.png"></noscript></div></div><figcaption class="imageCaption"><a href="https://skymind.ai/wiki/bagofwords-tf-idf" data-href="https://skymind.ai/wiki/bagofwords-tf-idf" class="markup--anchor markup--figure-anchor" rel="nofollow noopener" target="_blank">https://skymind.ai/wiki/bagofwords-tf-idf</a></figcaption></figure><p name="9ed6" id="9ed6" class="graf graf--p graf-after--figure">However even though tf-idf BoW representations provide weights to different words they are unable to capture the word meaning.</p><p name="3746" id="3746" class="graf graf--p graf-after--p">As the famous linguist J. R. Firth said in 1935, “<em class="markup--em markup--p-em">The complete meaning of a word is always contextual, and no study of meaning apart from context can be taken seriously.</em>”</p><p name="7b75" id="7b75" class="graf graf--p graf-after--p"><strong class="markup--strong markup--p-strong">Distributional Embeddings </strong>enable word vectors to encapsulate contextual context. Each embedding vector is represented based on the mutual information it has with other words in a given corpus. Mutual information can be represented as a global co-occurrence frequency or restricted to a given window either sequentially or based on dependency edges.</p><figure name="07c9" id="07c9" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 262px; max-height: 265px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 101.1%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="1*5oi3GdoCsk84JQGIVFmeAQ.png" data-width="262" data-height="265" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_5oi3GdoCsk84JQGIVFmeAQ.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="75"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/1*5oi3GdoCsk84JQGIVFmeAQ.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_5oi3GdoCsk84JQGIVFmeAQ(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/1*5oi3GdoCsk84JQGIVFmeAQ.png"></noscript></div></div><figcaption class="imageCaption">An example distributional embedding matrix each row encodes distributional context based on the count of the words it co-occurs with</figcaption></figure><p name="4a02" id="4a02" class="graf graf--p graf-after--figure graf--trailing">Distributional vectors predate neural methods for word embeddings and the techniques surrounding them are still relevant as they provide insight into better interpreting what neural embeddings learn. For more information one should read the work of <a href="http://www.aclweb.org/anthology/Q15-1016" data-href="http://www.aclweb.org/anthology/Q15-1016" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">Goldberg and Levy</a>.</p></div></div></section><section name="c2ba" class="section section--body"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><h3 name="16b1" id="16b1" class="graf graf--h3 graf--leading"><strong class="markup--strong markup--h3-strong">Neural Embeddings</strong></h3><p name="3bca" id="3bca" class="graf graf--p graf-after--h3"><a href="https://machinelearningmastery.com/develop-word-embeddings-python-gensim/" data-href="https://machinelearningmastery.com/develop-word-embeddings-python-gensim/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><strong class="markup--strong markup--p-strong">Word2Vec</strong></a></p><p name="3d70" id="3d70" class="graf graf--p graf-after--p"><span class="markup--quote markup--p-quote is-other" name="anon_b20aefd002af" data-creator-ids="anon">Predictive models learn their vectors in order to improve their predictive ability of a loss such as the loss of predicting the vector for a target word from the vectors of the surrounding context words.</span></p><p name="21d0" id="21d0" class="graf graf--p graf-after--p">Word2Vec is a predictive embedding model. There are two main Word2Vec architectures that are used to produce a <a href="https://en.wikipedia.org/wiki/Distributed_representation" data-href="https://en.wikipedia.org/wiki/Distributed_representation" class="markup--anchor markup--p-anchor" title="Distributed representation" rel="noopener" target="_blank">distributed representation</a> of words:</p><ul class="postList"><li name="b203" id="b203" class="graf graf--li graf-after--p"><a href="https://en.wikipedia.org/wiki/Bag-of-words_model#CBOW" data-href="https://en.wikipedia.org/wiki/Bag-of-words_model#CBOW" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Continuous bag-of-words</a> (CBOW) — The order of context words does not influence prediction (<a href="https://en.wikipedia.org/wiki/Bag-of-words" data-href="https://en.wikipedia.org/wiki/Bag-of-words" class="markup--anchor markup--li-anchor" title="Bag-of-words" rel="noopener" target="_blank">bag-of-words</a> assumption). In the continuous skip-gram architecture, the model uses the current word to predict the surrounding window of context words.</li><li name="b425" id="b425" class="graf graf--li graf-after--li"><a href="https://en.wikipedia.org/wiki/N-gram#Skip-gram" data-href="https://en.wikipedia.org/wiki/N-gram#Skip-gram" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Continuous skip-gram</a> weighs nearby context words more heavily than more distant context words. While order still is not captured each of the context vectors are weighed and compared independently vs CBOW which weighs against the average context.</li></ul><figure name="6b59" id="6b59" class="graf graf--figure graf-after--li"><div class="aspectRatioPlaceholder is-locked" style="max-width: 680px; max-height: 286px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 42.1%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*TY9nYgPpwJloevhp.png" data-width="680" data-height="286" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_TY9nYgPpwJloevhp.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="31"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*TY9nYgPpwJloevhp.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_TY9nYgPpwJloevhp(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*TY9nYgPpwJloevhp.png"></noscript></div></div><figcaption class="imageCaption">CBOW and Skip-Gram Architectures</figcaption></figure><p name="849c" id="849c" class="graf graf--p graf-after--figure">CBOW is faster while skip-gram is slower but does a better job for infrequent words.</p><p name="684d" id="684d" class="graf graf--p graf-after--p"><a href="https://nlp.stanford.edu/projects/glove/" data-href="https://nlp.stanford.edu/projects/glove/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><strong class="markup--strong markup--p-strong">GloVe</strong></a></p><p name="1056" id="1056" class="graf graf--p graf-after--p">Both CBOW and Skip-Grams are “predictive” models, in that they <strong class="markup--strong markup--p-strong">only take local contexts into account</strong>. Word2Vec does not take advantage of global context. <a href="https://nlp.stanford.edu/projects/glove/" data-href="https://nlp.stanford.edu/projects/glove/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">GloVe</a><strong class="markup--strong markup--p-strong"> </strong>embeddings by contrast leverage the same intuition behind the co-occuring matrix used distributional embeddings, but uses neural methods to decompose the co-occurrence matrix into more expressive and dense word vectors. While GloVe vectors are faster to train, neither GloVe or Word2Vec has been shown to provide definitively better results rather they should both be evaluated for a given dataset.</p><p name="4c3d" id="4c3d" class="graf graf--p graf-after--p"><a href="https://fasttext.cc/" data-href="https://fasttext.cc/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><strong class="markup--strong markup--p-strong">FastText</strong></a></p><p name="fb95" id="fb95" class="graf graf--p graf-after--p graf--trailing">FastText, builds on Word2Vec by learning vector representations for each word and the n-grams found within each word. The values of the representations are then averaged into one vector at each training step. While this adds a lot of additional computation to training it enables word embeddings to encode sub-word information. FastText vectors have been shown to be more accurate than Word2Vec vectors by a number of different measures</p></div></div></section><section name="76fb" class="section section--body section--last"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><h3 name="d4f5" id="d4f5" class="graf graf--h3 graf--leading">A 10,000 foot overview of Neural NLP Architectures</h3><p name="4573" id="4573" class="graf graf--p graf-after--h3">In addition to better word vector representation the advent of neural has led to advances in machine learning architectures that have enabled the advances listed in the <a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" class="markup--anchor markup--p-anchor" target="_blank">previous post</a>.</p><p name="0f21" id="0f21" class="graf graf--p graf-after--p">This section will highlight some of the key developments in neural architecture that enabled some of the NLP advances seen thus far. This not meant to be an exhaustive review of deep learning and machine learning NLP architecture, rather the goal is to demonstrate the changes that are driving NLP forward.</p><h4 name="b1de" id="b1de" class="graf graf--h4 graf-after--p">Deep Feed Forward&nbsp;Networks</h4><figure name="eaf7" id="eaf7" class="graf graf--figure graf-after--h4"><div class="aspectRatioPlaceholder"><img class="graf-image" data-image-id="1*5CsWEdiDbInS2eZxgU3vKg.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_5CsWEdiDbInS2eZxgU3vKg.png"></div></figure><p name="5fc6" id="5fc6" class="graf graf--p graf-after--figure">The advent of linear deep feed forward networks also known as <a href="https://en.wikipedia.org/wiki/Multilayer_perceptron" data-href="https://en.wikipedia.org/wiki/Multilayer_perceptron" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">multi layer perceptrons (MLP) </a>in NLP introduced the potential for non linear modeling. This development helps with NLP because there are cases where the embedding space may be non linear. Take the following example of a documents whose embedding space is non linear meaning there is no way to linear divide the two document groups.</p><figure name="3313" id="3313" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 209px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 29.799999999999997%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*R4cLnK1LdTUsxBuR.gif" data-width="1106" data-height="330" data-action="zoom" data-action-value="0*R4cLnK1LdTUsxBuR.gif" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_R4cLnK1LdTUsxBuR.gif" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="21"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*R4cLnK1LdTUsxBuR.gif" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_R4cLnK1LdTUsxBuR(1).gif"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*R4cLnK1LdTUsxBuR.gif"></noscript></div></div><figcaption class="imageCaption">It doesn’t matter how you fit a line there is no linear way to split the spam and ham documents</figcaption></figure><p name="5542" id="5542" class="graf graf--p graf-after--figure">A non linear MLP network provides the ability to properly model such non linearities.</p><figure name="e2bf" id="e2bf" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 203px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 28.999999999999996%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-imageLoaded is-canvasLoaded" data-image-id="1*Q7JqxCpm-V8Pi69xtE12mA.png" data-width="884" data-height="256" data-action="zoom" data-action-value="1*Q7JqxCpm-V8Pi69xtE12mA.png" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_Q7JqxCpm-V8Pi69xtE12mA.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="21"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/1*Q7JqxCpm-V8Pi69xtE12mA.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_Q7JqxCpm-V8Pi69xtE12mA(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/1*Q7JqxCpm-V8Pi69xtE12mA.png"></noscript></div></div></figure><p name="818f" id="818f" class="graf graf--p graf-after--figure">This development by itself however did not bring about a significant revolution in NLP, since MLPs are unable to model word ordering. While MLPs open the door for marginal improvements in tasks such as language classification, where decisions can be made by modeling independent character frequencies, for more complex or ambiguous tasks standalone MLPs fall short.</p><h4 name="a812" id="a812" class="graf graf--h4 graf-after--p">1D CNNs</h4><figure name="a757" id="a757" class="graf graf--figure graf-after--h4"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 282px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 40.300000000000004%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*J3WBLXd8yFg8MAZp.png" data-width="1600" data-height="645" data-action="zoom" data-action-value="0*J3WBLXd8yFg8MAZp.png" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_J3WBLXd8yFg8MAZp.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="30"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*J3WBLXd8yFg8MAZp.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_J3WBLXd8yFg8MAZp(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*J3WBLXd8yFg8MAZp.png"></noscript></div></div><figcaption class="imageCaption"><em class="markup--em markup--figure-em">Kim, Y. (2014). Convolutional Neural Networks for Sentence Classification</em></figcaption></figure><p name="0b8f" id="0b8f" class="graf graf--p graf-after--figure">Prior to their application in NLP <a href="https://en.wikipedia.org/wiki/Convolutional_neural_network" data-href="https://en.wikipedia.org/wiki/Convolutional_neural_network" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">Convolutional Neural Networks (CNNs) </a>provided groundbreaking results computer vision with the advent of <a href="https://en.wikipedia.org/wiki/AlexNet" data-href="https://en.wikipedia.org/wiki/AlexNet" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">AlexNet</a> In NLP instead of convolving over pixels convulsion filters are applied and pooled sequentially over individual or groups of word vectors</p><p name="aa21" id="aa21" class="graf graf--p graf-after--p">In NLP CNNs are able to model local ordering by acting as <strong class="markup--strong markup--p-strong"><em class="markup--em markup--p-em">n-gram feature extractors for embeddings</em></strong>. CNN models have contributed to state of the art results in classification and a variety of other NLP tasks.</p><p name="8c33" id="8c33" class="graf graf--p graf-after--p">More recently the work of <a href="https://arxiv.org/abs/1809.08037" data-href="https://arxiv.org/abs/1809.08037" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">Jacovi and Golberg et al</a>, has contributed to deeper understanding of what convolutional filters learn by demonstrating that filters are able to model rich semantic classes of n-grams by using different activation patterns, and that global max-pooling induces behavior which filters out less relevant n-grams from model decision process.</p><p name="5817" id="5817" class="graf graf--p graf-after--p">A good primer on getting started with 1D CNNs can be found in the embedded link below.</p><div name="efb2" id="efb2" class="graf graf--mixtapeEmbed graf-after--p"><a href="https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3" data-href="https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3" class="markup--anchor markup--mixtapeEmbed-anchor" title="https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3"><strong class="markup--strong markup--mixtapeEmbed-strong">Deep Learning for Natural Language Processing — Part III</strong><br><em class="markup--em markup--mixtapeEmbed-em">It’s been a month since I wrote the first part of this series. There, I shared the bit I know about word vector…</em>medium.com</a><a href="https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3" class="js-mixtapeImage mixtapeImage u-ignoreBlock" data-media-id="5b437c70f6fe532baaa11bd673585973" data-thumbnail-img-id="1*_tssg04hrYZlM6Ys4pUckw.png" style="background-image: url(https://cdn-images-1.medium.com/fit/c/320/320/1*_tssg04hrYZlM6Ys4pUckw.png);"></a></div><h4 name="7fe9" id="7fe9" class="graf graf--h4 graf-after--mixtapeEmbed">RNNs (LSTM/GRU)</h4><p name="4d74" id="4d74" class="graf graf--p graf-after--h4">Building on the local ordering provide by CNNs Recurrent Neural Networks (RNNs) and their gated cell variants such as Long Short Term Memory Cells (LSTMs) and Gated Recurrent Units (GRUs) provide mechanisms for modeling sequential ordering and mid range dependencies in text such as the affect of a word in the beginning of a sentence on the end of a sentence.</p><div name="27d1" id="27d1" class="graf graf--mixtapeEmbed graf-after--p"><a href="https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21" data-href="https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21" class="markup--anchor markup--mixtapeEmbed-anchor" title="https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21"><strong class="markup--strong markup--mixtapeEmbed-strong">Illustrated Guide to LSTM’s and GRU’s: A step by step explanation</strong><br><em class="markup--em markup--mixtapeEmbed-em">Hi and welcome to an Illustrated Guide to Long Short-Term Memory (LSTM) and Gated Recurrent Units (GRU). I’m Michael…</em>towardsdatascience.com</a><a href="https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21" class="js-mixtapeImage mixtapeImage u-ignoreBlock" data-media-id="3ce932bc502ce1fc9c6e50626db58deb" data-thumbnail-img-id="1*n-IgHZM5baBUjq0T7RYDBw.gif" style="background-image: url(https://cdn-images-1.medium.com/fit/c/320/320/1*n-IgHZM5baBUjq0T7RYDBw.gif);"></a></div><p name="b031" id="b031" class="graf graf--p graf-after--mixtapeEmbed">Additional variations of RNNs such as <a href="https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks" data-href="https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">Bidirectional-RNNs</a> which process text in both left to right and right to left and <a href="https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html" data-href="https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">character level RNNs </a>for enhancing underrepresented or out of vocabulary word embeddings led to many state of the art neural <a href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/" data-href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">NLP breakthroughs</a>.</p><figure name="0a60" id="0a60" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 394px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 56.2%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*MvzSPj3wF-nelK0j.gif" data-width="2667" data-height="1500" data-action="zoom" data-action-value="0*MvzSPj3wF-nelK0j.gif" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_MvzSPj3wF-nelK0j.gif" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="41"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*MvzSPj3wF-nelK0j.gif" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_MvzSPj3wF-nelK0j(1).gif"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*MvzSPj3wF-nelK0j.gif"></noscript></div></div><figcaption class="imageCaption">An sample of some different RNN architectures and coupled with example use&nbsp;cases.</figcaption></figure><h4 name="3f3f" id="3f3f" class="graf graf--h4 graf-after--figure">Attention and Copy Mechanisms</h4><p name="dafe" id="dafe" class="graf graf--p graf-after--h4">While standard RNN architectures have led to incredible <a href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/" data-href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">breakthroughs</a> in NLP they suffer from a variety of challenges. While in theory they can capture long term dependencies they tend to struggle modeling longer sequences, this is still an open problem.</p><p name="a65f" id="a65f" class="graf graf--p graf-after--p">One cause for sub-optimal performance standard RNN encoder-decoder models for sequence to sequence tasks such as <a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" class="markup--anchor markup--p-anchor" target="_blank">NER</a> or translation is that they weight the impact each input vector evenly on each output vector when in reality specific words in the input sequence may carry more importance at different time steps.</p><p name="deb7" id="deb7" class="graf graf--p graf-after--p"><a href="https://skymind.ai/wiki/attention-mechanism-memory-network" data-href="https://skymind.ai/wiki/attention-mechanism-memory-network" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><strong class="markup--strong markup--p-strong">Attention mechanisms</strong></a> provide a means of weighting the contextual impact of each input vector on each output prediction of the RNN. These mechanisms are responsible for much of the current or near current state of the art in Natural language processing.</p><figure name="e771" id="e771" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 678px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 96.8%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*bVTfAB5K6aDSy1PG.gif" data-width="755" data-height="731" data-action="zoom" data-action-value="0*bVTfAB5K6aDSy1PG.gif" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_bVTfAB5K6aDSy1PG.gif" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="72"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*bVTfAB5K6aDSy1PG.gif" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_bVTfAB5K6aDSy1PG(1).gif"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*bVTfAB5K6aDSy1PG.gif"></noscript></div></div><figcaption class="imageCaption">An example of an attention mechanism applied to the task of neural translation in Microsoft Translator</figcaption></figure><p name="7246" id="7246" class="graf graf--p graf-after--figure">Additionally in <a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" class="markup--anchor markup--p-anchor" target="_blank">Machine Reading Comprehension</a> and <a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f" class="markup--anchor markup--p-anchor" target="_blank">Summarization</a> systems RNNs often tend to generate results, that while on first glance look structurally correct are in reality hallucinated or incorrect. One mechanism that helps mitigate some of these issues is the <a href="http://www.abigailsee.com/2017/04/16/taming-rnns-for-better-summarization.html" data-href="http://www.abigailsee.com/2017/04/16/taming-rnns-for-better-summarization.html" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank"><strong class="markup--strong markup--p-strong">Copy Mechanism</strong></a><strong class="markup--strong markup--p-strong">.</strong></p><figure name="e30f" id="e30f" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 700px; max-height: 457px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 65.3%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="0*onHSVW8HkDma_EN2.png" data-width="1226" data-height="800" data-action="zoom" data-action-value="0*onHSVW8HkDma_EN2.png" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_onHSVW8HkDma_EN2.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="48"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/0*onHSVW8HkDma_EN2.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_onHSVW8HkDma_EN2(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/0*onHSVW8HkDma_EN2.png"></noscript></div></div><figcaption class="imageCaption">Copy Mechanism from Get To The Point: Summarization with Pointer-Generator Networks Abigail See, et&nbsp;all</figcaption></figure><p name="bdba" id="bdba" class="graf graf--p graf-after--figure">The copy mechanism is an additional layer applied during decoding that decides whether it is better to generate the next word from the source sentence or from the general embedding vocabulary.</p><h4 name="7c14" id="7c14" class="graf graf--h4 graf-after--p">Putting it all together with ELMo and&nbsp;BERT</h4><p name="fdb6" id="fdb6" class="graf graf--p graf-after--h4"><a href="https://allennlp.org/elmo" data-href="https://allennlp.org/elmo" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">ELMo</a> is a model generates embeddings for a word based on the context it appears thus generating slightly different embeddings for each of its occurrence.</p><figure name="05a4" id="05a4" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 580px; max-height: 318px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 54.800000000000004%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="1*euk-3hzyi9nJvTdWFmfrqQ.png" data-width="580" data-height="318" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_euk-3hzyi9nJvTdWFmfrqQ.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="40"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/1*euk-3hzyi9nJvTdWFmfrqQ.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_euk-3hzyi9nJvTdWFmfrqQ(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/1*euk-3hzyi9nJvTdWFmfrqQ.png"></noscript></div></div></figure><p name="89ec" id="89ec" class="graf graf--p graf-after--figure">For example, the word “<strong class="markup--strong markup--p-strong"><em class="markup--em markup--p-em">play</em></strong>” in the sentence above using standard word embeddings encodes multiple meanings such as the verb <strong class="markup--strong markup--p-strong"><em class="markup--em markup--p-em">to play</em></strong> or in the case of the sentence a theatre production. In standard word embeddings such as Glove, Fast Text or Word2Vec each instance of the word <strong class="markup--strong markup--p-strong"><em class="markup--em markup--p-em">play</em></strong><em class="markup--em markup--p-em"> </em>would have the same representation.</p><p name="1576" id="1576" class="graf graf--p graf-after--p">ELMo enables NLP models to better disambiguate between the correct sense of a given word. On in it’s release it enabled near instant state of the art results in many downstream tasks, including tasks such as co-reference were previously not as viable for practical usage.</p><figure name="8151" id="8151" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 556px; max-height: 442px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 79.5%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="1*BlrJnsOP_TxX1mgfT81I0Q.png" data-width="556" data-height="442" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_BlrJnsOP_TxX1mgfT81I0Q.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="58"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/1*BlrJnsOP_TxX1mgfT81I0Q.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_BlrJnsOP_TxX1mgfT81I0Q(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/1*BlrJnsOP_TxX1mgfT81I0Q.png"></noscript></div></div></figure><p name="4f97" id="4f97" class="graf graf--p graf-after--figure">ELMo also provides promising implications for preforming transfer learning on out of domain datasets. Some such as Sebastien Ruder have even hailed the coming ELMo as the <a href="http://ruder.io/nlp-imagenet/" data-href="http://ruder.io/nlp-imagenet/" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">ImageNet moment of NLP</a> and while ELMo is a very promising development with practical real world applications, and has spawned recent related techniques such as <a href="https://arxiv.org/pdf/1810.04805.pdf" data-href="https://arxiv.org/pdf/1810.04805.pdf" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">BERT</a>, that use attention transformers instead of bi-directonal RNNs to encode context, we will see in our upcoming post that there are still many obstacles in the world of Neural NLP.</p><figure name="9ed7" id="9ed7" class="graf graf--figure graf-after--p"><div class="aspectRatioPlaceholder is-locked" style="max-width: 617px; max-height: 240px;"><div class="aspectRatioPlaceholder-fill" style="padding-bottom: 38.9%;"></div><div class="progressiveMedia js-progressiveMedia graf-image is-canvasLoaded is-imageLoaded" data-image-id="1*8WhXg3oXUC4s-m7F2ePLEA.png" data-width="617" data-height="240" data-scroll="native"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_8WhXg3oXUC4s-m7F2ePLEA.png" crossorigin="anonymous" class="progressiveMedia-thumbnail js-progressiveMedia-thumbnail"><canvas class="progressiveMedia-canvas js-progressiveMedia-canvas" width="75" height="28"></canvas><img class="progressiveMedia-image js-progressiveMedia-image" data-src="https://cdn-images-1.medium.com/max/1600/1*8WhXg3oXUC4s-m7F2ePLEA.png" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_8WhXg3oXUC4s-m7F2ePLEA(1).png"><noscript class="js-progressiveMedia-inner"><img class="progressiveMedia-noscript js-progressiveMedia-inner" src="https://cdn-images-1.medium.com/max/1600/1*8WhXg3oXUC4s-m7F2ePLEA.png"></noscript></div></div><figcaption class="imageCaption">Comparsion of <a href="https://arxiv.org/pdf/1810.04805.pdf" data-href="https://arxiv.org/pdf/1810.04805.pdf" class="markup--anchor markup--figure-anchor" rel="noopener" target="_blank">BERT</a> and ELMo architectures from Devlin et.&nbsp;all</figcaption></figure><h3 name="e042" id="e042" class="graf graf--h3 graf-after--figure">Call To Action: Getting&nbsp;Started</h3><p name="7773" id="7773" class="graf graf--p graf-after--h3">Below are some resources to get started with the the different word embeddings above.</p><p name="3f9f" id="3f9f" class="graf graf--p graf-after--p"><strong class="markup--strong markup--p-strong">Documentation</strong></p><ul class="postList"><li name="daef" id="daef" class="graf graf--li graf-after--p"><a href="https://docs.microsoft.com/en-us/learn/modules/interactive-deep-learning/?WT.mc_id=blog-medium-abornst" data-href="https://docs.microsoft.com/en-us/learn/modules/interactive-deep-learning/?WT.mc_id=blog-medium-abornst" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Getting started with pyTorch and Docker on the Azure DLVM</a></li><li name="bd6e" id="bd6e" class="graf graf--li graf-after--li"><a href="https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html" data-href="https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Character RNN classification with pyTorch</a></li><li name="6da8" id="6da8" class="graf graf--li graf-after--li"><a href="https://fasttext.cc/docs/en/supervised-tutorial.html" data-href="https://fasttext.cc/docs/en/supervised-tutorial.html" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Fast Text Tutorial</a></li><li name="2a4a" id="2a4a" class="graf graf--li graf-after--li"><a href="https://nlpforhackers.io/keras-intro/" data-href="https://nlpforhackers.io/keras-intro/" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Keras NLP intro</a></li></ul><p name="1345" id="1345" class="graf graf--p graf-after--li"><strong class="markup--strong markup--p-strong">Tools</strong></p><ul class="postList"><li name="6106" id="6106" class="graf graf--li graf-after--p"><a href="https://radimrehurek.com/gensim/" data-href="https://radimrehurek.com/gensim/" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Gensim</a></li><li name="af81" id="af81" class="graf graf--li graf-after--li"><a href="https://radimrehurek.com/gensim/" data-href="https://radimrehurek.com/gensim/" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Word2Vec</a></li><li name="daaf" id="daaf" class="graf graf--li graf-after--li"><a href="http://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html" data-href="http://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Sci-Kit Learn Bag of Words</a></li><li name="7fe6" id="7fe6" class="graf graf--li graf-after--li"><a href="https://spacy.io/usage/examples" data-href="https://spacy.io/usage/examples" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">SpaCy Examples</a></li><li name="78d9" id="78d9" class="graf graf--li graf-after--li"><a href="https://github.com/pytorch/text" data-href="https://github.com/pytorch/text" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">PyTorch Text</a></li><li name="50dd" id="50dd" class="graf graf--li graf-after--li"><a href="https://github.com/allenai/allennlp/blob/master/tutorials/how_to/elmo.md" data-href="https://github.com/allenai/allennlp/blob/master/tutorials/how_to/elmo.md" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Allen NLP ELMo</a></li><li name="9e9a" id="9e9a" class="graf graf--li graf-after--li"><a href="https://github.com/huggingface/pytorch-pretrained-BERT" data-href="https://github.com/huggingface/pytorch-pretrained-BERT" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Hugging Face BERT PyTorch</a></li><li name="47d5" id="47d5" class="graf graf--li graf-after--li"><a href="https://github.com/keon/awesome-nlp" data-href="https://github.com/keon/awesome-nlp" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Additional Resources</a></li></ul><p name="3fb9" id="3fb9" class="graf graf--p graf-after--li"><strong class="markup--strong markup--p-strong">Open Dataset</strong></p><ul class="postList"><li name="ad2d" id="ad2d" class="graf graf--li graf-after--p"><a href="https://msropendata.com/datasets/30a504b0-cff2-4d4a-864f-3bc9a66f9d7e" data-href="https://msropendata.com/datasets/30a504b0-cff2-4d4a-864f-3bc9a66f9d7e" class="markup--anchor markup--li-anchor" rel="noopener" target="_blank">Dual Word Embeddings Trained on Bing Queries</a></li></ul><h4 name="c155" id="c155" class="graf graf--h4 graf-after--li"><a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0" class="markup--anchor markup--h4-anchor" target="_blank">Next Post</a></h4><p name="36e1" id="36e1" class="graf graf--p graf-after--h4">Now that we have a solid understanding of some of the milestones in neural NLP, as well as the models and representations in the <a href="https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0" data-href="https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0" class="markup--anchor markup--p-anchor" target="_blank">next post will review some of the pitfalls of current state of the art NLP systems</a>.</p><p name="6b3c" id="6b3c" class="graf graf--p graf-after--p">If you have any questions, comments, or topics you would like me to discuss feel free to follow me on <a href="https://twitter.com/pythiccoder" data-href="https://twitter.com/pythiccoder" class="markup--anchor markup--p-anchor" rel="noopener" target="_blank">Twitter</a>.</p><p name="9137" id="9137" class="graf graf--p graf-after--p graf--trailing"><em class="markup--em markup--p-em">About the Author<br></em>Aaron (Ari) Bornstein is an avid AI enthusiast with a passion for history, engaging with new technologies and computational medicine. As an Open Source Engineer at Microsoft’s Cloud Developer Advocacy team, he collaborates with Israeli Hi-Tech Community, to solve real world problems with game changing technologies that are then documented, open sourced, and shared with the rest of the world.</p></div></div></section></div><footer class="u-paddingTop10"><div class="container u-maxWidth740"><div class="row"><div class="col u-size12of12"><div class="postMetaInline postMetaInline--acknowledgments u-paddingTop5 u-paddingBottom20 js-postMetaAcknowledgments"><span data-tooltip="The following people helped the author by providing feedback before the story was published.">Thanks to</span> <span><a class="link u-baseColor--link" href="https://medium.com/@adipolak?source=post_page" data-action="show-user-card" data-action-source="post_page" data-action-value="46b9b20c06bd" data-action-type="hover" data-user-id="46b9b20c06bd" dir="auto">Adi Polak</a></span>. </div></div></div><div class="row"><div class="col u-size12of12 js-postTags"><div class="u-paddingBottom10"><ul class="tags tags--postTags tags--borderless"><li><a class="link u-baseColor--link" href="https://towardsdatascience.com/tagged/machine-learning?source=post" data-action-source="post" data-collection-slug="towards-data-science">Machine Learning</a></li><li><a class="link u-baseColor--link" href="https://towardsdatascience.com/tagged/nlp?source=post" data-action-source="post" data-collection-slug="towards-data-science">NLP</a></li><li><a class="link u-baseColor--link" href="https://towardsdatascience.com/tagged/ai?source=post" data-action-source="post" data-collection-slug="towards-data-science">AI</a></li><li><a class="link u-baseColor--link" href="https://towardsdatascience.com/tagged/deep-learning?source=post" data-action-source="post" data-collection-slug="towards-data-science">Deep Learning</a></li><li><a class="link u-baseColor--link" href="https://towardsdatascience.com/tagged/data-science?source=post" data-action-source="post" data-collection-slug="towards-data-science">Data Science</a></li></ul></div></div></div><div class="postActions js-postActionsFooter "><div class="u-flexCenter"><div class="u-flex1"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="4ebd4711d0ec" data-is-icon-29px="true" data-is-circle="true" data-has-recommend-list="true" data-source="post_actions_footer-----4ebd4711d0ec---------------------clap_footer" data-clap-string-singular="clap" data-clap-string-plural="claps"><div class="u-relative u-foreground"><button class="button button--large button--circle button--withChrome u-baseColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker clapButton--largePill u-relative u-foreground u-xs-paddingLeft13 u-width60 u-height60 u-accentColor--textNormal u-accentColor--buttonNormal clap-onboardingcollection" data-action="multivote" data-action-value="4ebd4711d0ec" data-action-type="long-press" data-action-source="post_actions_footer-----4ebd4711d0ec---------------------clap_footer" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--33px u-relative u-topNegative2 u-xs-top0"><svg class="svgIcon-use" width="33" height="33"><path d="M28.86 17.342l-3.64-6.402c-.292-.433-.712-.729-1.163-.8a1.124 1.124 0 0 0-.889.213c-.63.488-.742 1.181-.33 2.061l1.222 2.587 1.4 2.46c2.234 4.085 1.511 8.007-2.145 11.663-.26.26-.526.49-.797.707 1.42-.084 2.881-.683 4.292-2.094 3.822-3.823 3.565-7.876 2.05-10.395zm-6.252 11.075c3.352-3.35 3.998-6.775 1.978-10.469l-3.378-5.945c-.292-.432-.712-.728-1.163-.8a1.122 1.122 0 0 0-.89.213c-.63.49-.742 1.182-.33 2.061l1.72 3.638a.502.502 0 0 1-.806.568l-8.91-8.91a1.335 1.335 0 0 0-1.887 1.886l5.292 5.292a.5.5 0 0 1-.707.707l-5.292-5.292-1.492-1.492c-.503-.503-1.382-.505-1.887 0a1.337 1.337 0 0 0 0 1.886l1.493 1.492 5.292 5.292a.499.499 0 0 1-.353.854.5.5 0 0 1-.354-.147L5.642 13.96a1.338 1.338 0 0 0-1.887 0 1.338 1.338 0 0 0 0 1.887l2.23 2.228 3.322 3.324a.499.499 0 0 1-.353.853.502.502 0 0 1-.354-.146l-3.323-3.324a1.333 1.333 0 0 0-1.886 0 1.325 1.325 0 0 0-.39.943c0 .356.138.691.39.943l6.396 6.397c3.528 3.53 8.86 5.313 12.821 1.353zM12.73 9.26l5.68 5.68-.49-1.037c-.518-1.107-.426-2.13.224-2.89l-3.303-3.304a1.337 1.337 0 0 0-1.886 0 1.326 1.326 0 0 0-.39.944c0 .217.067.42.165.607zm14.787 19.184c-1.599 1.6-3.417 2.392-5.353 2.392-.349 0-.7-.03-1.058-.082a7.922 7.922 0 0 1-3.667.887c-3.049 0-6.115-1.626-8.359-3.87l-6.396-6.397A2.315 2.315 0 0 1 2 19.724a2.327 2.327 0 0 1 1.923-2.296l-.875-.875a2.339 2.339 0 0 1 0-3.3 2.33 2.33 0 0 1 1.24-.647l-.139-.139c-.91-.91-.91-2.39 0-3.3.884-.884 2.421-.882 3.301 0l.138.14a2.335 2.335 0 0 1 3.948-1.24l.093.092c.091-.423.291-.828.62-1.157a2.336 2.336 0 0 1 3.3 0l3.384 3.386a2.167 2.167 0 0 1 1.271-.173c.534.086 1.03.354 1.441.765.11-.549.415-1.034.911-1.418a2.12 2.12 0 0 1 1.661-.41c.727.117 1.385.565 1.853 1.262l3.652 6.423c1.704 2.832 2.025 7.377-2.205 11.607zM13.217.484l-1.917.882 2.37 2.837-.454-3.719zm8.487.877l-1.928-.86-.44 3.697 2.368-2.837zM16.5 3.293L15.478-.005h2.044L16.5 3.293z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--33px u-relative u-topNegative2 u-xs-top0"><svg class="svgIcon-use" width="33" height="33"><g fill-rule="evenodd"><path d="M29.58 17.1l-3.854-6.78c-.365-.543-.876-.899-1.431-.989a1.491 1.491 0 0 0-1.16.281c-.42.327-.65.736-.7 1.207v.001l3.623 6.367c2.46 4.498 1.67 8.802-2.333 12.807-.265.265-.536.505-.81.728 1.973-.222 3.474-1.286 4.45-2.263 4.166-4.165 3.875-8.6 2.215-11.36zm-4.831.82l-3.581-6.3c-.296-.439-.725-.742-1.183-.815a1.105 1.105 0 0 0-.89.213c-.647.502-.755 1.188-.33 2.098l1.825 3.858a.601.601 0 0 1-.197.747.596.596 0 0 1-.77-.067L10.178 8.21c-.508-.506-1.393-.506-1.901 0a1.335 1.335 0 0 0-.393.95c0 .36.139.698.393.95v.001l5.61 5.61a.599.599 0 1 1-.848.847l-5.606-5.606c-.001 0-.002 0-.003-.002L5.848 9.375a1.349 1.349 0 0 0-1.902 0 1.348 1.348 0 0 0 0 1.901l1.582 1.582 5.61 5.61a.6.6 0 0 1-.848.848l-5.61-5.61c-.51-.508-1.393-.508-1.9 0a1.332 1.332 0 0 0-.394.95c0 .36.139.697.393.952l2.363 2.362c.002.001.002.002.002.003l3.52 3.52a.6.6 0 0 1-.848.847l-3.522-3.523h-.001a1.336 1.336 0 0 0-.95-.393 1.345 1.345 0 0 0-.949 2.295l6.779 6.78c3.715 3.713 9.327 5.598 13.49 1.434 3.527-3.528 4.21-7.13 2.086-11.015zM11.817 7.727c.06-.328.213-.64.466-.893.64-.64 1.755-.64 2.396 0l3.232 3.232c-.82.783-1.09 1.833-.764 2.992l-5.33-5.33z"></path><path d="M13.285.48l-1.916.881 2.37 2.837z"></path><path d="M21.719 1.361L19.79.501l-.44 3.697z"></path><path d="M16.502 3.298L15.481 0h2.043z"></path></g></svg></span></span></button><div class="clapUndo u-width60 u-round u-height32 u-absolute u-borderBox u-paddingRight5 u-transition--transform200Springu-backgroundGrayLighter js-clapUndo" style="top: 14px; padding: 2px;"><button class="button button--chromeless u-baseColor--buttonNormal button--withIcon button--withSvgIcon u-floatRight" data-action="multivote-undo" data-action-value="4ebd4711d0ec"><span class="svgIcon svgIcon--removeThin svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M20.13 8.11l-5.61 5.61-5.609-5.61-.801.801 5.61 5.61-5.61 5.61.801.8 5.61-5.609 5.61 5.61.8-.801-5.609-5.61 5.61-5.61" fill-rule="evenodd"></path></svg></span></button></div></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft16"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-textColorDarker" data-action="show-recommends" data-action-value="4ebd4711d0ec">1.93K claps</button><span class="u-xs-hide"></span></span></div></div><div class="buttonSet u-flex0"><a class="button button--dark button--chromeless u-baseColor--buttonDark button--withIcon button--withSvgIcon button--dark button--chromeless u-xs-hide u-marginRight12" href="https://medium.com/p/4ebd4711d0ec/share/twitter" title="Share on Twitter" aria-label="Share on Twitter" target="_blank" data-action-source="post_actions_footer"><span class="button-defaultState"><span class="svgIcon svgIcon--twitterFilled svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M22.053 7.54a4.474 4.474 0 0 0-3.31-1.455 4.526 4.526 0 0 0-4.526 4.524c0 .35.04.7.082 1.05a12.9 12.9 0 0 1-9.3-4.77c-.39.69-.61 1.46-.65 2.26.03 1.6.83 2.99 2.02 3.79-.72-.02-1.41-.22-2.02-.57-.01.02-.01.04 0 .08-.01 2.17 1.55 4 3.63 4.44-.39.08-.79.13-1.21.16-.28-.03-.57-.05-.81-.08.54 1.77 2.21 3.08 4.2 3.15a9.564 9.564 0 0 1-5.66 1.94c-.34-.03-.7-.06-1.05-.08 2 1.27 4.38 2.02 6.94 2.02 8.31 0 12.86-6.9 12.84-12.85.02-.24.01-.43 0-.65.89-.62 1.65-1.42 2.26-2.34-.82.38-1.69.62-2.59.72a4.37 4.37 0 0 0 1.94-2.51c-.84.53-1.81.9-2.83 1.13z"></path></svg></span></span></a><a class="button button--dark button--chromeless u-baseColor--buttonDark button--withIcon button--withSvgIcon button--dark button--chromeless u-xs-hide u-marginRight12" href="https://medium.com/p/4ebd4711d0ec/share/facebook" title="Share on Facebook" aria-label="Share on Facebook" target="_blank" data-action-source="post_actions_footer"><span class="button-defaultState"><span class="svgIcon svgIcon--facebookSquare svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M23.209 5H5.792A.792.792 0 0 0 5 5.791V23.21c0 .437.354.791.792.791h9.303v-7.125H12.72v-2.968h2.375v-2.375c0-2.455 1.553-3.662 3.741-3.662 1.049 0 1.95.078 2.213.112v2.565h-1.517c-1.192 0-1.469.567-1.469 1.397v1.963h2.969l-.594 2.968h-2.375L18.11 24h5.099a.791.791 0 0 0 .791-.791V5.79a.791.791 0 0 0-.791-.79"></path></svg></span></span></a><button class="button button--large button--dark button--chromeless u-baseColor--buttonDark button--withIcon button--withSvgIcon u-xs-show u-marginRight10" title="Share this story on Twitter or Facebook" aria-label="Share this story on Twitter or Facebook" data-action="show-share-popover" data-action-source="post_actions_footer"><span class="svgIcon svgIcon--share svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M20.385 8H19a.5.5 0 1 0 .011 1h1.39c.43 0 .84.168 1.14.473.31.305.48.71.48 1.142v10.77c0 .43-.17.837-.47 1.142-.3.305-.71.473-1.14.473H8.62c-.43 0-.84-.168-1.144-.473a1.603 1.603 0 0 1-.473-1.142v-10.77c0-.43.17-.837.48-1.142A1.599 1.599 0 0 1 8.62 9H10a.502.502 0 0 0 0-1H8.615c-.67 0-1.338.255-1.85.766-.51.51-.765 1.18-.765 1.85v10.77c0 .668.255 1.337.766 1.848.51.51 1.18.766 1.85.766h11.77c.668 0 1.337-.255 1.848-.766.51-.51.766-1.18.766-1.85v-10.77c0-.668-.255-1.337-.766-1.848A2.61 2.61 0 0 0 20.384 8zm-8.67-2.508L14 3.207v8.362c0 .27.224.5.5.5s.5-.23.5-.5V3.2l2.285 2.285a.49.49 0 0 0 .704-.001.511.511 0 0 0 0-.708l-3.14-3.14a.504.504 0 0 0-.71 0L11 4.776a.501.501 0 0 0 .71.706" fill-rule="evenodd"></path></svg></span></button><button class="button button--large button--dark button--chromeless is-touchIconBlackPulse u-baseColor--buttonDark button--withIcon button--withSvgIcon" data-action="respond" data-action-source="post_actions_footer"><span class="svgIcon svgIcon--response svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M21.27 20.058c1.89-1.826 2.754-4.17 2.754-6.674C24.024 8.21 19.67 4 14.1 4 8.53 4 4 8.21 4 13.384c0 5.175 4.53 9.385 10.1 9.385 1.007 0 2-.14 2.95-.41.285.25.592.49.918.7 1.306.87 2.716 1.31 4.19 1.31.276-.01.494-.14.6-.36a.625.625 0 0 0-.052-.65c-.61-.84-1.042-1.71-1.282-2.58a5.417 5.417 0 0 1-.154-.75zm-3.85 1.324l-.083-.28-.388.12a9.72 9.72 0 0 1-2.85.424c-4.96 0-8.99-3.706-8.99-8.262 0-4.556 4.03-8.263 8.99-8.263 4.95 0 8.77 3.71 8.77 8.27 0 2.25-.75 4.35-2.5 5.92l-.24.21v.32c0 .07 0 .19.02.37.03.29.1.6.19.92.19.7.49 1.4.89 2.08-.93-.14-1.83-.49-2.67-1.06-.34-.22-.88-.48-1.16-.74z"></path></svg></span></button><button class="button button--chromeless u-baseColor--buttonNormal u-marginRight12" data-action="scroll-to-responses">4</button><button class="button button--large button--dark button--chromeless is-touchIconFadeInPulse u-baseColor--buttonDark button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="4ebd4711d0ec" data-action-source="post_actions_footer"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--29px u-marginRight4"><svg class="svgIcon-use" width="29" height="29"><path d="M19.385 4h-9.77A2.623 2.623 0 0 0 7 6.615V23.01a1.022 1.022 0 0 0 1.595.847l5.905-4.004 5.905 4.004A1.022 1.022 0 0 0 22 23.011V6.62A2.625 2.625 0 0 0 19.385 4zM21 23l-5.91-3.955-.148-.107a.751.751 0 0 0-.884 0l-.147.107L8 23V6.615C8 5.725 8.725 5 9.615 5h9.77C20.275 5 21 5.725 21 6.615V23z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--29px u-marginRight4"><svg class="svgIcon-use" width="29" height="29"><path d="M19.385 4h-9.77A2.623 2.623 0 0 0 7 6.615V23.01a1.022 1.022 0 0 0 1.595.847l5.905-4.004 5.905 4.004A1.022 1.022 0 0 0 22 23.011V6.62A2.625 2.625 0 0 0 19.385 4z" fill-rule="evenodd"></path></svg></span></span></button><button class="button button--large button--dark button--chromeless is-touchIconBlackPulse u-baseColor--buttonDark button--withIcon button--withSvgIcon js-moreActionsButton" title="More actions" aria-label="More actions" data-action="more-actions"><span class="svgIcon svgIcon--more svgIcon--25px"><svg class="svgIcon-use" width="25" height="25" viewBox="-480.5 272.5 21 21"><path d="M-463 284.6c.9 0 1.6-.7 1.6-1.6s-.7-1.6-1.6-1.6-1.6.7-1.6 1.6.7 1.6 1.6 1.6zm0 .9c-1.4 0-2.5-1.1-2.5-2.5s1.1-2.5 2.5-2.5 2.5 1.1 2.5 2.5-1.1 2.5-2.5 2.5zm-7-.9c.9 0 1.6-.7 1.6-1.6s-.7-1.6-1.6-1.6-1.6.7-1.6 1.6.7 1.6 1.6 1.6zm0 .9c-1.4 0-2.5-1.1-2.5-2.5s1.1-2.5 2.5-2.5 2.5 1.1 2.5 2.5-1.1 2.5-2.5 2.5zm-7-.9c.9 0 1.6-.7 1.6-1.6s-.7-1.6-1.6-1.6-1.6.7-1.6 1.6.7 1.6 1.6 1.6zm0 .9c-1.4 0-2.5-1.1-2.5-2.5s1.1-2.5 2.5-2.5 2.5 1.1 2.5 2.5-1.1 2.5-2.5 2.5z"></path></svg></span></button></div></div></div></div><div class="u-maxWidth740 u-paddingTop20 u-marginTop20 u-borderTopLightest container u-paddingBottom20 u-xs-paddingBottom10 js-postAttributionFooterContainer"><div class="row js-postFooterInfo"><div class="col u-size6of12 u-xs-size12of12"><li class="uiScale uiScale-ui--small uiScale-caption--regular u-block u-paddingBottom18 js-cardUser"><div class="u-marginLeft20 u-floatRight"><span class="followState js-followState" data-user-id="b3c7769e3e2f"><button class="button button--small u-noUserSelect button--withChrome u-baseColor--buttonNormal button--withHover button--unblock js-unblockButton" data-action="toggle-block-user" data-action-value="b3c7769e3e2f" data-action-source="footer_card"><span class="button-label  button-defaultState">Blocked</span><span class="button-label button-hoverState">Unblock</span></button><button class="button button--primary button--small u-noUserSelect button--withChrome u-accentColor--buttonNormal button--follow js-followButton" data-action="toggle-subscribe-user" data-action-value="b3c7769e3e2f" data-action-source="footer_card-b3c7769e3e2f-------------------------follow_footer" data-subscribe-source="footer_card" data-follow-context-entity-id="4ebd4711d0ec"><span class="button-label  button-defaultState js-buttonLabel">Follow</span><span class="button-label button-activeState">Following</span></button></span></div><div class="u-tableCell"><a class="link u-baseColor--link avatar" href="https://towardsdatascience.com/@aribornstein?source=footer_card" title="Go to the profile of Aaron (Ari) Bornstein" aria-label="Go to the profile of Aaron (Ari) Bornstein" data-action-source="footer_card" data-user-id="b3c7769e3e2f" data-collection-slug="towards-data-science" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_9Y0zWLwh1nYuBZMetDnC7w(1).jpeg" class="avatar-image avatar-image--small" alt="Go to the profile of Aaron (Ari) Bornstein"></a></div><div class="u-tableCell u-verticalAlignMiddle u-breakWord u-paddingLeft15"><h3 class="ui-h3 u-fontSize18 u-lineHeightTighter u-marginBottom4"><a class="link link--primary u-accentColor--hoverTextNormal" href="https://towardsdatascience.com/@aribornstein" property="cc:attributionName" title="Go to the profile of Aaron (Ari) Bornstein" aria-label="Go to the profile of Aaron (Ari) Bornstein" rel="author cc:attributionUrl" data-user-id="b3c7769e3e2f" data-collection-slug="towards-data-science" dir="auto">Aaron (Ari) Bornstein</a></h3><p class="ui-body u-fontSize14 u-lineHeightBaseSans u-textColorDark u-marginBottom4">&lt;Microsoft Open Source Engineer&gt; I am an AI enthusiast with a passion for engaging with new technologies, history, and computational medicine.</p></div></li></div><div class="col u-size6of12 u-xs-size12of12 u-xs-marginTop30"><li class="uiScale uiScale-ui--small uiScale-caption--regular u-block u-paddingBottom18 js-cardCollection"><div class="u-marginLeft20 u-floatRight"><button class="button button--primary button--small u-noUserSelect button--withChrome u-accentColor--buttonNormal js-relationshipButton" data-action="toggle-follow-collection" data-action-source="footer_card----7f60cf5620c9----------------------follow_footer" data-collection-id="7f60cf5620c9"><span class="button-label  js-buttonLabel">Follow</span></button></div><div class="u-tableCell "><a class="link u-baseColor--link avatar avatar--roundedRectangle" href="https://towardsdatascience.com/?source=footer_card" title="Go to Towards Data Science" aria-label="Go to Towards Data Science" data-action-source="footer_card" data-collection-slug="towards-data-science"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_F0LADxTtsKOgmPa-_7iUEQ.jpeg" class="avatar-image u-size60x60" alt="Towards Data Science"></a></div><div class="u-tableCell u-verticalAlignMiddle u-breakWord u-paddingLeft15"><h3 class="ui-h3 u-fontSize18 u-lineHeightTighter u-marginBottom4"><a class="link link--primary u-accentColor--hoverTextNormal" href="https://towardsdatascience.com/?source=footer_card" rel="collection" data-action-source="footer_card" data-collection-slug="towards-data-science">Towards Data Science</a></h3><p class="ui-body u-fontSize14 u-lineHeightBaseSans u-textColorDark u-marginBottom4">Sharing concepts, ideas, and codes.</p><div class="buttonSet"></div></div></li></div></div></div><div class="js-postFooterPlacements" data-post-id="4ebd4711d0ec" data-collection-id="7f60cf5620c9" data-scroll="native"><div class="streamItem streamItem--placementCardGrid js-streamItem"><div class="u-clearfix u-backgroundGrayLightest"><div class="row u-marginAuto u-maxWidth1032 u-paddingTop30 u-paddingBottom40"><div class="col u-padding8 u-xs-size12of12 u-size4of12"><div class="uiScale uiScale-ui--small uiScale-caption--regular u-height280 u-width100pct u-backgroundWhite u-borderCardBorder u-boxShadow u-borderBox u-borderRadius4 js-trackPostPresentation" data-post-id="1a0d73f15012" data-source="placement_card_footer_grid---------0-41" data-tracking-context="placement" data-scroll="native"><a class="link link--noUnderline u-baseColor--link" href="https://towardsdatascience.com/why-youre-not-a-job-ready-data-scientist-yet-1a0d73f15012?source=placement_card_footer_grid---------0-41" data-action-source="placement_card_footer_grid---------0-41"><div class="u-backgroundCover u-backgroundColorGrayLight u-height100 u-width100pct u-borderBottomLight u-borderRadiusTop4" style="background-image: url(&quot;https://cdn-images-1.medium.com/fit/c/400/120/0*8F8p5yS5x1OtGdCe.jpg&quot;); background-position: 50% 50% !important;"></div></a><div class="u-padding15 u-borderBox u-flexColumn u-height180"><a class="link link--noUnderline u-baseColor--link u-flex1" href="https://towardsdatascience.com/why-youre-not-a-job-ready-data-scientist-yet-1a0d73f15012?source=placement_card_footer_grid---------0-41" data-action-source="placement_card_footer_grid---------0-41"><div class="uiScale uiScale-ui--regular uiScale-caption--small u-textColorNormal u-marginBottom7">More from Towards Data Science</div><div class="ui-h3 ui-clamp2 u-textColorDarkest u-contentSansBold u-fontSize24 u-maxHeight2LineHeightTighter u-lineClamp2 u-textOverflowEllipsis u-letterSpacingTight u-paddingBottom2">Why you’re not a job-ready data scientist (yet)</div></a><div class="u-paddingBottom10 u-flex0 u-flexCenter"><div class="u-flex1 u-minWidth0 u-marginRight10"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://towardsdatascience.com/@jeremie_sharpestminds" data-action="show-user-card" data-action-value="59564831d1eb" data-action-type="hover" data-user-id="59564831d1eb" data-collection-slug="towards-data-science" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_xciaYUDHbs-SwfMn4TGCRw.jpeg" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of Jeremie Harris"></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--darker" href="https://towardsdatascience.com/@jeremie_sharpestminds?source=placement_card_footer_grid---------0-41" data-action="show-user-card" data-action-source="placement_card_footer_grid---------0-41" data-action-value="59564831d1eb" data-action-type="hover" data-user-id="59564831d1eb" data-collection-slug="towards-data-science" dir="auto">Jeremie Harris</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://towardsdatascience.com/why-youre-not-a-job-ready-data-scientist-yet-1a0d73f15012?source=placement_card_footer_grid---------0-41" data-action="open-post" data-action-value="https://towardsdatascience.com/why-youre-not-a-job-ready-data-scientist-yet-1a0d73f15012?source=placement_card_footer_grid---------0-41" data-action-source="preview-listing"><time datetime="2019-06-16T22:18:26.747Z">Jun 17</time></a><span class="middotDivider u-fontSize12"></span><span class="readingTime" title="6 min read"></span></div></div></div></div><div class="u-flex0 u-flexCenter"><div class="buttonSet"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="1a0d73f15012" data-is-label-padded="true" data-source="placement_card_footer_grid-----1a0d73f15012----0-41----------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="1a0d73f15012" data-action-type="long-press" data-action-source="placement_card_footer_grid-----1a0d73f15012----0-41----------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-disablePointerEvents u-marginLeft4" data-action="show-recommends" data-action-value="1a0d73f15012">6.1K</button></span></div></div><div class="u-height20 u-borderRightLighter u-inlineBlock u-relative u-marginRight10 u-marginLeft12"></div><div class="buttonSet"><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="1a0d73f15012" data-action-source="placement_card_footer_grid-----1a0d73f15012----0-41----------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button></div></div></div></div></div></div><div class="col u-padding8 u-xs-size12of12 u-size4of12"><div class="uiScale uiScale-ui--small uiScale-caption--regular u-height280 u-width100pct u-backgroundWhite u-borderCardBorder u-boxShadow u-borderBox u-borderRadius4 js-trackPostPresentation" data-post-id="ec18c6396e6b" data-source="placement_card_footer_grid---------1-41" data-tracking-context="placement" data-scroll="native"><a class="link link--noUnderline u-baseColor--link" href="https://towardsdatascience.com/10-simple-hacks-to-speed-up-your-data-analysis-in-python-ec18c6396e6b?source=placement_card_footer_grid---------1-41" data-action-source="placement_card_footer_grid---------1-41"><div class="u-backgroundCover u-backgroundColorGrayLight u-height100 u-width100pct u-borderBottomLight u-borderRadiusTop4" style="background-image: url(&quot;https://cdn-images-1.medium.com/fit/c/400/120/1*E6ZU7TRjiPNpFEdxuQ7WmA.jpeg&quot;); background-position: 50% 50% !important;"></div></a><div class="u-padding15 u-borderBox u-flexColumn u-height180"><a class="link link--noUnderline u-baseColor--link u-flex1" href="https://towardsdatascience.com/10-simple-hacks-to-speed-up-your-data-analysis-in-python-ec18c6396e6b?source=placement_card_footer_grid---------1-41" data-action-source="placement_card_footer_grid---------1-41"><div class="uiScale uiScale-ui--regular uiScale-caption--small u-textColorNormal u-marginBottom7">More from Towards Data Science</div><div class="ui-h3 ui-clamp2 u-textColorDarkest u-contentSansBold u-fontSize24 u-maxHeight2LineHeightTighter u-lineClamp2 u-textOverflowEllipsis u-letterSpacingTight u-paddingBottom2">10 Simple hacks to speed up your Data Analysis in Python</div></a><div class="u-paddingBottom10 u-flex0 u-flexCenter"><div class="u-flex1 u-minWidth0 u-marginRight10"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://towardsdatascience.com/@parulnith" data-action="show-user-card" data-action-value="7053de462a28" data-action-type="hover" data-user-id="7053de462a28" data-collection-slug="towards-data-science" dir="auto"><div class="u-relative u-inlineBlock u-flex0"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_-ooorT2_5GQSfQoVFxJHXw.jpeg" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of Parul Pandey"><div class="avatar-halo u-absolute u-textColorGreenNormal svgIcon" style="width: calc(100% + 10px); height: calc(100% + 10px); top:-5px; left:-5px"><svg viewBox="0 0 40 40" xmlns="http://www.w3.org/2000/svg"><path d="M3.44615311,11.6601601 C6.57294867,5.47967718 12.9131553,1.5 19.9642857,1.5 C27.0154162,1.5 33.3556228,5.47967718 36.4824183,11.6601601 L37.3747245,11.2087295 C34.0793076,4.69494641 27.3961457,0.5 19.9642857,0.5 C12.5324257,0.5 5.84926381,4.69494641 2.55384689,11.2087295 L3.44615311,11.6601601 Z"></path><path d="M36.4824183,28.2564276 C33.3556228,34.4369105 27.0154162,38.4165876 19.9642857,38.4165876 C12.9131553,38.4165876 6.57294867,34.4369105 3.44615311,28.2564276 L2.55384689,28.7078582 C5.84926381,35.2216412 12.5324257,39.4165876 19.9642857,39.4165876 C27.3961457,39.4165876 34.0793076,35.2216412 37.3747245,28.7078582 L36.4824183,28.2564276 Z"></path></svg></div></div></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--darker" href="https://towardsdatascience.com/@parulnith?source=placement_card_footer_grid---------1-41" data-action="show-user-card" data-action-source="placement_card_footer_grid---------1-41" data-action-value="7053de462a28" data-action-type="hover" data-user-id="7053de462a28" data-collection-slug="towards-data-science" dir="auto">Parul Pandey</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://towardsdatascience.com/10-simple-hacks-to-speed-up-your-data-analysis-in-python-ec18c6396e6b?source=placement_card_footer_grid---------1-41" data-action="open-post" data-action-value="https://towardsdatascience.com/10-simple-hacks-to-speed-up-your-data-analysis-in-python-ec18c6396e6b?source=placement_card_footer_grid---------1-41" data-action-source="preview-listing"><time datetime="2019-06-17T14:29:01.790Z">Jun 17</time></a><span class="middotDivider u-fontSize12"></span><span class="readingTime" title="8 min read"></span></div></div></div></div><div class="u-flex0 u-flexCenter"><div class="buttonSet"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="ec18c6396e6b" data-is-label-padded="true" data-source="placement_card_footer_grid-----ec18c6396e6b----1-41----------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="ec18c6396e6b" data-action-type="long-press" data-action-source="placement_card_footer_grid-----ec18c6396e6b----1-41----------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-disablePointerEvents u-marginLeft4" data-action="show-recommends" data-action-value="ec18c6396e6b">4.3K</button></span></div></div><div class="u-height20 u-borderRightLighter u-inlineBlock u-relative u-marginRight10 u-marginLeft12"></div><div class="buttonSet"><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="ec18c6396e6b" data-action-source="placement_card_footer_grid-----ec18c6396e6b----1-41----------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button></div></div></div></div></div></div><div class="col u-padding8 u-xs-size12of12 u-size4of12"><div class="uiScale uiScale-ui--small uiScale-caption--regular u-height280 u-width100pct u-backgroundWhite u-borderCardBorder u-boxShadow u-borderBox u-borderRadius4 js-trackPostPresentation" data-post-id="ac35326219e4" data-source="placement_card_footer_grid---------2-41" data-tracking-context="placement" data-scroll="native"><a class="link link--noUnderline u-baseColor--link" href="https://towardsdatascience.com/what-70-of-data-science-learners-do-wrong-ac35326219e4?source=placement_card_footer_grid---------2-41" data-action-source="placement_card_footer_grid---------2-41"><div class="u-backgroundCover u-backgroundColorGrayLight u-height100 u-width100pct u-borderBottomLight u-borderRadiusTop4" style="background-image: url(&quot;https://cdn-images-1.medium.com/fit/c/400/120/0*5-jyf1BubNO-Vm6u&quot;); background-position: 50% 50% !important;"></div></a><div class="u-padding15 u-borderBox u-flexColumn u-height180"><a class="link link--noUnderline u-baseColor--link u-flex1" href="https://towardsdatascience.com/what-70-of-data-science-learners-do-wrong-ac35326219e4?source=placement_card_footer_grid---------2-41" data-action-source="placement_card_footer_grid---------2-41"><div class="uiScale uiScale-ui--regular uiScale-caption--small u-textColorNormal u-marginBottom7">More from Towards Data Science</div><div class="ui-h3 ui-clamp2 u-textColorDarkest u-contentSansBold u-fontSize24 u-maxHeight2LineHeightTighter u-lineClamp2 u-textOverflowEllipsis u-letterSpacingTight u-paddingBottom2">What 70% of Data Science Learners Do Wrong</div></a><div class="u-paddingBottom10 u-flex0 u-flexCenter"><div class="u-flex1 u-minWidth0 u-marginRight10"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://towardsdatascience.com/@dansbecker" data-action="show-user-card" data-action-value="1e98ffe425cd" data-action-type="hover" data-user-id="1e98ffe425cd" data-collection-slug="towards-data-science" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_8e9JtU3h04zrUXD6" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of Dan Becker"></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--darker" href="https://towardsdatascience.com/@dansbecker?source=placement_card_footer_grid---------2-41" data-action="show-user-card" data-action-source="placement_card_footer_grid---------2-41" data-action-value="1e98ffe425cd" data-action-type="hover" data-user-id="1e98ffe425cd" data-collection-slug="towards-data-science" dir="auto">Dan Becker</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://towardsdatascience.com/what-70-of-data-science-learners-do-wrong-ac35326219e4?source=placement_card_footer_grid---------2-41" data-action="open-post" data-action-value="https://towardsdatascience.com/what-70-of-data-science-learners-do-wrong-ac35326219e4?source=placement_card_footer_grid---------2-41" data-action-source="preview-listing"><time datetime="2019-06-14T13:23:18.218Z">Jun 14</time></a><span class="middotDivider u-fontSize12"></span><span class="readingTime" title="3 min read"></span></div></div></div></div><div class="u-flex0 u-flexCenter"><div class="buttonSet"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="ac35326219e4" data-is-label-padded="true" data-source="placement_card_footer_grid-----ac35326219e4----2-41----------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="ac35326219e4" data-action-type="long-press" data-action-source="placement_card_footer_grid-----ac35326219e4----2-41----------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-disablePointerEvents u-marginLeft4" data-action="show-recommends" data-action-value="ac35326219e4">3K</button></span></div></div><div class="u-height20 u-borderRightLighter u-inlineBlock u-relative u-marginRight10 u-marginLeft12"></div><div class="buttonSet"><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="ac35326219e4" data-action-source="placement_card_footer_grid-----ac35326219e4----2-41----------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button></div></div></div></div></div></div></div></div></div></div><div class="u-padding0 u-clearfix u-backgroundGrayLightest u-print-hide supplementalPostContent js-responsesWrapper" data-action-scope="_actionscope_5"><div class="container u-maxWidth740"><div class="responsesStreamWrapper u-maxWidth640 js-responsesStreamWrapper"><div class="container responsesStream-title u-paddingTop15"><div class="row"><header class="heading"><div class="u-clearfix"><div class="heading-content u-floatLeft"><span class="heading-title heading-title--semibold">Responses</span></div></div></header></div></div><div class="responsesStream-editor cardChromeless u-marginBottom20 u-paddingLeft20 u-paddingRight20 js-responsesStreamEditor"><div class="inlineNewPostControl js-inlineNewPostControl" data-action-scope="_actionscope_13"><div class="inlineEditor is-collapsed is-postEditMode js-inlineEditor" data-action="focus-editor"><div class="u-paddingTop20 js-block js-inlineEditorContent"><div class="inlineEditor-header"><div class="inlineEditor-avatar u-paddingRight20"><div class="avatar u-inline"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_Mpgm7EXaTEICpUOi(1).jpg" class="avatar-image u-size36x36 u-xs-size32x32" alt="Cheng-Jun Wang"></div></div><div class="inlineEditor-headerContent"><div class="inlineEditor-placeholder js-inlineEditorPrompt">Write a response…</div><div class="inlineEditor-author u-accentColor--textNormal">Cheng-Jun Wang</div></div></div></div></div></div></div><div class="responsesStream js-responsesStream"><div class="streamItem streamItem--postPreview js-streamItem" data-action-scope="_actionscope_6"><div class="cardChromeless u-marginTop20 u-paddingTop10 u-paddingBottom15 u-paddingLeft20 u-paddingRight20"><div class="postArticle postArticle--short js-postArticle js-trackPostPresentation js-trackPostScrolls" data-post-id="dee2954c2670" data-source="responses---------0-31----------------------" data-action-scope="_actionscope_7" data-scroll="native"><div class="u-marginBottom10"><div class="postMetaInline">Applause from <a class="link link--darken u-accentColor--textDarken u-baseColor--link" href="https://medium.com/@aribornstein" data-action="show-user-card" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" dir="auto">Aaron (Ari) Bornstein</a> (author)</div></div><div class="u-clearfix u-marginBottom15 u-paddingTop5"><div class="postMetaInline u-floatLeft"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://medium.com/@1035194981" data-action="show-user-card" data-action-value="4b98d86ec0c1" data-action-type="hover" data-user-id="4b98d86ec0c1" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/0_dGOd4tIyB1GEPzeh" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of 张悦"></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--accent u-accentColor--textNormal u-accentColor--textDarken" href="https://medium.com/@1035194981?source=responses---------0-31----------------------" data-action="show-user-card" data-action-source="responses---------0-31----------------------" data-action-value="4b98d86ec0c1" data-action-type="hover" data-user-id="4b98d86ec0c1" dir="auto">张悦</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://medium.com/@1035194981/bow%E4%B8%8D%E8%83%BD%E7%BB%99%E5%87%BA%E8%AF%8D%E6%84%8F%E4%BF%A1%E6%81%AF-dee2954c2670?source=responses---------0-31----------------------" data-action="open-post" data-action-value="https://medium.com/@1035194981/bow%E4%B8%8D%E8%83%BD%E7%BB%99%E5%87%BA%E8%AF%8D%E6%84%8F%E4%BF%A1%E6%81%AF-dee2954c2670?source=responses---------0-31----------------------" data-action-source="preview-listing"><time datetime="2018-11-22T01:43:38.923Z">Nov 22, 2018</time></a></div></div></div></div></div><a href="https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec?source=responses---------0-31----------------------#7adf"><div class="u-fontSize14 u-marginTop10 u-marginBottom20 u-padding14 u-xs-padding12 u-borderRadius3 u-borderCardBackground u-borderLighterHover u-boxShadow1px4pxCardBorder"><div class="label label--quote u-accentColor--highlightFaint">One challenge with bag of word representations is that they don’t encode any information with regards to the meaning of a given word.</div></div></a><div><a class="" href="https://medium.com/@1035194981/bow%E4%B8%8D%E8%83%BD%E7%BB%99%E5%87%BA%E8%AF%8D%E6%84%8F%E4%BF%A1%E6%81%AF-dee2954c2670?source=responses---------0-31----------------------" data-action-source="responses---------0-31----------------------"><div class="postArticle-content js-postField"><section class="section section--body section--first section--last"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><p name="4070" id="4070" class="graf graf--p graf--leading graf--trailing">BOW不能给出词意信息。</p></div></div></section></div></a></div><div class="u-clearfix u-paddingTop10"><div class="u-floatLeft"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="dee2954c2670" data-is-flush-left="true" data-source="listing-----dee2954c2670---------------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="dee2954c2670" data-action-type="long-press" data-action-source="listing-----dee2954c2670---------------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-disablePointerEvents" data-action="show-recommends" data-action-value="dee2954c2670">5</button></span></div></div><div class="buttonSet u-floatRight"><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="dee2954c2670" data-action-source="listing-----dee2954c2670---------------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button><button class="button button--chromeless is-touchIconBlackPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon js-postActionsButton" data-action="post-actions" data-action-value="dee2954c2670"><span class="svgIcon svgIcon--arrowDown svgIcon--19px is-flushRight"><svg class="svgIcon-use" width="19" height="19"><path d="M3.9 6.772l5.205 5.756.427.472.427-.472 5.155-5.698-.854-.772-4.728 5.254L4.753 6z" fill-rule="evenodd"></path></svg></span></button></div></div></div></div></div><div class="streamItem streamItem--postPreview js-streamItem" data-action-scope="_actionscope_8"><div class="cardChromeless u-marginTop20 u-paddingTop10 u-paddingBottom15 u-paddingLeft20 u-paddingRight20"><div class="postArticle postArticle--short js-postArticle js-trackPostPresentation js-trackPostScrolls" data-post-id="453470135eb" data-source="responses---------1-31----------------------" data-action-scope="_actionscope_9" data-scroll="native"><div class="u-marginBottom10"><div class="postMetaInline">Applause from <a class="link link--darken u-accentColor--textDarken u-baseColor--link" href="https://medium.com/@aribornstein" data-action="show-user-card" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" dir="auto">Aaron (Ari) Bornstein</a> (author)</div></div><div class="u-clearfix u-marginBottom15 u-paddingTop5"><div class="postMetaInline u-floatLeft"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://medium.com/@hoomanjfr" data-action="show-user-card" data-action-value="bca351d4a6dc" data-action-type="hover" data-user-id="bca351d4a6dc" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_xI8rJwmLMExVYPuxhyr5Fw.jpeg" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of Hooman Jafari"></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--accent u-accentColor--textNormal u-accentColor--textDarken" href="https://medium.com/@hoomanjfr?source=responses---------1-31----------------------" data-action="show-user-card" data-action-source="responses---------1-31----------------------" data-action-value="bca351d4a6dc" data-action-type="hover" data-user-id="bca351d4a6dc" dir="auto">Hooman Jafari</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://medium.com/@hoomanjfr/thanks-a-lot-453470135eb?source=responses---------1-31----------------------" data-action="open-post" data-action-value="https://medium.com/@hoomanjfr/thanks-a-lot-453470135eb?source=responses---------1-31----------------------" data-action-source="preview-listing"><time datetime="2018-10-19T08:11:58.100Z">Oct 19, 2018</time></a></div></div></div></div></div><div><a class="" href="https://medium.com/@hoomanjfr/thanks-a-lot-453470135eb?source=responses---------1-31----------------------" data-action-source="responses---------1-31----------------------"><div class="postArticle-content js-postField"><section class="section section--body section--first section--last"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><p name="045d" id="045d" class="graf graf--p graf--leading graf--trailing">Thanks a lot&nbsp;!</p></div></div></section></div></a></div><div class="u-clearfix u-paddingTop10"><div class="u-floatLeft"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="453470135eb" data-is-flush-left="true" data-source="listing-----453470135eb---------------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="453470135eb" data-action-type="long-press" data-action-source="listing-----453470135eb---------------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-disablePointerEvents" data-action="show-recommends" data-action-value="453470135eb">1</button></span></div></div><div class="buttonSet u-floatRight"><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="453470135eb" data-action-source="listing-----453470135eb---------------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button><button class="button button--chromeless is-touchIconBlackPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon js-postActionsButton" data-action="post-actions" data-action-value="453470135eb"><span class="svgIcon svgIcon--arrowDown svgIcon--19px is-flushRight"><svg class="svgIcon-use" width="19" height="19"><path d="M3.9 6.772l5.205 5.756.427.472.427-.472 5.155-5.698-.854-.772-4.728 5.254L4.753 6z" fill-rule="evenodd"></path></svg></span></button></div></div></div></div></div><div class="streamItem streamItem--conversation js-streamItem" data-action-scope="_actionscope_10"><div class="streamItemConversation"><div class="u-marginLeft20"><div class="streamItemConversation-divider"></div><header class="heading heading--light heading--simple"><div class="u-clearfix"><div class="heading-content u-floatLeft"><span class="heading-title">Conversation with <a class="link link--accent u-accentColor--textNormal u-baseColor--link" href="https://medium.com/@aribornstein" data-action="show-user-card" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" dir="auto">Aaron (Ari) Bornstein</a>.</span></div></div></header></div><div class="streamItemConversation-inner cardChromeless"><div class="streamItemConversationItem streamItemConversationItem--preview"><div class="postArticle js-postArticle js-trackPostPresentation js-trackPostScrolls postArticle--short" data-post-id="acdfffb31bfd" data-source="responses---------2-----------------------" data-action-scope="_actionscope_11" data-scroll="native"><div class="u-clearfix u-marginBottom15 u-paddingTop5"><div class="postMetaInline u-floatLeft"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://medium.com/@bgoncalves" data-action="show-user-card" data-action-value="399f54e955da" data-action-type="hover" data-user-id="399f54e955da" dir="auto"><div class="u-relative u-inlineBlock u-flex0"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_BROjfAbXeRDU5QM4Fp5i6w.jpeg" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of Bruno Gonçalves"><div class="avatar-halo u-absolute u-textColorGreenNormal svgIcon" style="width: calc(100% + 10px); height: calc(100% + 10px); top:-5px; left:-5px"><svg viewBox="0 0 40 40" xmlns="http://www.w3.org/2000/svg"><path d="M3.44615311,11.6601601 C6.57294867,5.47967718 12.9131553,1.5 19.9642857,1.5 C27.0154162,1.5 33.3556228,5.47967718 36.4824183,11.6601601 L37.3747245,11.2087295 C34.0793076,4.69494641 27.3961457,0.5 19.9642857,0.5 C12.5324257,0.5 5.84926381,4.69494641 2.55384689,11.2087295 L3.44615311,11.6601601 Z"></path><path d="M36.4824183,28.2564276 C33.3556228,34.4369105 27.0154162,38.4165876 19.9642857,38.4165876 C12.9131553,38.4165876 6.57294867,34.4369105 3.44615311,28.2564276 L2.55384689,28.7078582 C5.84926381,35.2216412 12.5324257,39.4165876 19.9642857,39.4165876 C27.3961457,39.4165876 34.0793076,35.2216412 37.3747245,28.7078582 L36.4824183,28.2564276 Z"></path></svg></div></div></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--accent u-accentColor--textNormal u-accentColor--textDarken" href="https://medium.com/@bgoncalves?source=responses---------2-----------------------" data-action="show-user-card" data-action-source="responses---------2-----------------------" data-action-value="399f54e955da" data-action-type="hover" data-user-id="399f54e955da" dir="auto">Bruno Gonçalves</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://medium.com/@bgoncalves/thank-you-for-the-awesome-explanation-what-did-you-use-to-make-the-gifs-they-look-amazing-acdfffb31bfd?source=responses---------2-----------------------" data-action="open-post" data-action-value="https://medium.com/@bgoncalves/thank-you-for-the-awesome-explanation-what-did-you-use-to-make-the-gifs-they-look-amazing-acdfffb31bfd?source=responses---------2-----------------------" data-action-source="preview-listing"><time datetime="2018-11-24T17:49:08.923Z">Nov 25, 2018</time></a></div></div></div></div></div><div><a class="" href="https://medium.com/@bgoncalves/thank-you-for-the-awesome-explanation-what-did-you-use-to-make-the-gifs-they-look-amazing-acdfffb31bfd?source=responses---------2-----------------------"><div class="postArticle-content js-postField"><section class="section section--body section--first section--last"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><p name="91a0" id="91a0" class="graf graf--p graf--leading graf--trailing">Thank you for the awesome explanation. What did you use to make the gifs? They look amazing!</p></div></div></section></div></a></div><div class="u-clearfix u-paddingTop10"><div class="u-floatLeft"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="acdfffb31bfd" data-is-flush-left="true" data-source="listing-----acdfffb31bfd---------------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="acdfffb31bfd" data-action-type="long-press" data-action-source="listing-----acdfffb31bfd---------------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"></span></div></div><div class="buttonSet u-floatRight"><a class="button button--chromeless u-baseColor--buttonNormal" href="https://medium.com/@bgoncalves/thank-you-for-the-awesome-explanation-what-did-you-use-to-make-the-gifs-they-look-amazing-acdfffb31bfd?source=responses---------2-----------------------#--responses" data-action-source="responses---------2-----------------------">1 response</a><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="acdfffb31bfd" data-action-source="listing-----acdfffb31bfd---------------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button><button class="button button--chromeless is-touchIconBlackPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon js-postActionsButton" data-action="post-actions" data-action-value="acdfffb31bfd"><span class="svgIcon svgIcon--arrowDown svgIcon--19px is-flushRight"><svg class="svgIcon-use" width="19" height="19"><path d="M3.9 6.772l5.205 5.756.427.472.427-.472 5.155-5.698-.854-.772-4.728 5.254L4.753 6z" fill-rule="evenodd"></path></svg></span></button></div></div></div></div><div class="streamItemConversationItem streamItemConversationItem--preview"><div class="postArticle js-postArticle js-trackPostPresentation js-trackPostScrolls postArticle--short" data-post-id="b0e7a6227f76" data-source="responses---------2-----------------------" data-action-scope="_actionscope_12" data-scroll="native"><div class="u-clearfix u-marginBottom15 u-paddingTop5"><div class="postMetaInline u-floatLeft"><div class="u-flexCenter"><div class="postMetaInline-avatar u-flex0"><a class="link u-baseColor--link avatar" href="https://medium.com/@aribornstein" data-action="show-user-card" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" dir="auto"><img src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/1_9Y0zWLwh1nYuBZMetDnC7w(2).jpeg" class="avatar-image u-size36x36 u-xs-size32x32" alt="Go to the profile of Aaron (Ari) Bornstein"></a></div><div class="postMetaInline postMetaInline-authorLockup ui-captionStrong u-flex1 u-noWrapWithEllipsis"><a class="ds-link ds-link--styleSubtle link link--darken link--accent u-accentColor--textNormal u-accentColor--textDarken" href="https://medium.com/@aribornstein?source=responses---------2-----------------------" data-action="show-user-card" data-action-source="responses---------2-----------------------" data-action-value="b3c7769e3e2f" data-action-type="hover" data-user-id="b3c7769e3e2f" dir="auto">Aaron (Ari) Bornstein</a><div class="ui-caption u-fontSize12 u-baseColor--textNormal u-textColorNormal js-postMetaInlineSupplemental"><a class="link link--darken" href="https://medium.com/@aribornstein/for-this-post-i-used-power-point-you-can-save-slides-as-gif-b0e7a6227f76?source=responses---------2-----------------------" data-action="open-post" data-action-value="https://medium.com/@aribornstein/for-this-post-i-used-power-point-you-can-save-slides-as-gif-b0e7a6227f76?source=responses---------2-----------------------" data-action-source="preview-listing"><time datetime="2018-11-24T20:42:53.211Z">Nov 25, 2018</time></a></div></div></div></div></div><div><a class="" href="https://medium.com/@aribornstein/for-this-post-i-used-power-point-you-can-save-slides-as-gif-b0e7a6227f76?source=responses---------2-----------------------"><div class="postArticle-content js-postField"><section class="section section--body section--first section--last"><div class="section-divider"><hr class="section-divider"></div><div class="section-content"><div class="section-inner sectionLayout--insetColumn"><p name="bf84" id="bf84" class="graf graf--p graf--leading graf--trailing">For this post I used power point you can save slides as gif. For most of my posts I use <span class="markup--anchor markup--p-anchor" data-action="open-inner-link" data-action-value="https://www.screentogif.com/">https://www.screentogif.com/</span> which I’ve found to be an amazing tool!</p></div></div></section></div></a></div><div class="u-clearfix u-paddingTop10"><div class="u-floatLeft"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="b0e7a6227f76" data-is-flush-left="true" data-source="listing-----b0e7a6227f76---------------------clap_preview"><div class="u-relative u-foreground"><button class="button button--primary button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="b0e7a6227f76" data-action-type="long-press" data-action-source="listing-----b0e7a6227f76---------------------clap_preview" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.739 0l.761 2.966L13.261 0z"></path><path d="M14.815 3.776l1.84-2.551-1.43-.471z"></path><path d="M8.378 1.224l1.84 2.551L9.81.753z"></path><path d="M20.382 21.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L4.11 14.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L6.43 8.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L18.628 14c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM10.99 5.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--25px is-flushLeft"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M11.738 0l.762 2.966L13.262 0z"></path><path d="M16.634 1.224l-1.432-.47-.408 3.022z"></path><path d="M9.79.754l-1.431.47 1.84 2.552z"></path><path d="M22.472 13.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M12.58 9.887c-.156-.83.096-1.569.692-2.142L10.78 5.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M15.812 9.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L7.2 6.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L5.046 8.54 3.802 7.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394L3.647 9.93l4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L2.89 10.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C19.74 19.8 20.271 17 18.62 13.982L15.812 9.04z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton u-disablePointerEvents" data-action="show-recommends" data-action-value="b0e7a6227f76">1</button></span></div></div><div class="buttonSet u-floatRight"><a class="button button--chromeless u-baseColor--buttonNormal" href="https://medium.com/@aribornstein/for-this-post-i-used-power-point-you-can-save-slides-as-gif-b0e7a6227f76?source=responses---------2-----------------------#--responses" data-action-source="responses---------2-----------------------">1 response</a><button class="button button--chromeless is-touchIconFadeInPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="b0e7a6227f76" data-action-source="listing-----b0e7a6227f76---------------------bookmark_preview"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126a.508.508 0 0 0 .708-.03.5.5 0 0 0 .118-.285H19V6zm-6.838 9.97L7 19.636V6c0-.55.45-1 1-1h9c.55 0 1 .45 1 1v13.637l-5.162-3.668a.49.49 0 0 0-.676 0z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19 6c0-1.1-.9-2-2-2H8c-1.1 0-2 .9-2 2v14.66h.012c.01.103.045.204.12.285a.5.5 0 0 0 .706.03L12.5 16.85l5.662 4.126c.205.183.52.17.708-.03a.5.5 0 0 0 .118-.285H19V6z"></path></svg></span></span></button><button class="button button--chromeless is-touchIconBlackPulse u-baseColor--buttonNormal button--withIcon button--withSvgIcon js-postActionsButton" data-action="post-actions" data-action-value="b0e7a6227f76"><span class="svgIcon svgIcon--arrowDown svgIcon--19px is-flushRight"><svg class="svgIcon-use" width="19" height="19"><path d="M3.9 6.772l5.205 5.756.427.472.427-.472 5.155-5.698-.854-.772-4.728 5.254L4.753 6z" fill-rule="evenodd"></path></svg></span></button></div></div></div></div></div></div></div></div><div class="container js-showOtherResponses"><div class="row"><button class="button button--primary button--withChrome u-accentColor--buttonNormal responsesStream-showOtherResponses cardChromeless u-width100pct u-marginVertical20 u-heightAuto" data-action="show-other-responses">Show all responses</button></div></div><div class="responsesStream js-responsesStreamOther"></div></div></div></div><div class="supplementalPostContent js-heroPromo"></div></footer></article></main><aside class="u-marginAuto u-maxWidth1032 js-postLeftSidebar"><div class="u-foreground u-top0 u-sm-hide js-postShareWidget u-fixed u-transition--fadeOut300" data-scroll="fixed" style="transform: translateY(150px);"><div class="u-breakWord u-md-hide u-width131"><div class="u-width131 collection-title u-fontWeightBold u-fontSize18 u-lineHeightTight"><a href="https://towardsdatascience.com/?source=logo-3751a3493996">Towards Data Science</a></div><div class="u-width131 u-multiline-clamp u-textColorNormal u-fontSize14 u-lineHeightTight u-paddingTop3">Sharing concepts, ideas, and codes.</div><div class="u-paddingTop15 u-paddingBottom30 u-borderBottomLight u-marginBottom30"><button class="button button--primary button--small u-noUserSelect button--withChrome u-accentColor--buttonNormal js-relationshipButton" data-action="toggle-follow-collection" data-action-source="post_sidebar----7f60cf5620c9----------------------post_sidebar" data-collection-id="7f60cf5620c9"><span class="button-label  js-buttonLabel">Follow</span></button></div></div><ul><li class="u-marginVertical10"><div class="multirecommend js-actionMultirecommend u-flexCenter" data-post-id="4ebd4711d0ec" data-is-icon-29px="true" data-has-recommend-list="true" data-source="post_share_widget-----4ebd4711d0ec---------------------clap_sidebar"><div class="u-relative u-foreground"><button class="button button--primary button--large button--chromeless u-accentColor--buttonNormal button--withIcon button--withSvgIcon clapButton js-actionMultirecommendButton clapButton--darker" data-action="multivote" data-action-value="4ebd4711d0ec" data-action-type="long-press" data-action-source="post_share_widget-----4ebd4711d0ec---------------------clap_sidebar" aria-label="Clap"><span class="button-defaultState"><span class="svgIcon svgIcon--clap svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><g fill-rule="evenodd"><path d="M13.739 1l.761 2.966L15.261 1z"></path><path d="M16.815 4.776l1.84-2.551-1.43-.471z"></path><path d="M10.378 2.224l1.84 2.551-.408-3.022z"></path><path d="M22.382 22.622c-1.04 1.04-2.115 1.507-3.166 1.608.168-.14.332-.29.492-.45 2.885-2.886 3.456-5.982 1.69-9.211l-1.101-1.937-.955-2.02c-.315-.676-.235-1.185.245-1.556a.836.836 0 0 1 .66-.16c.342.056.66.28.879.605l2.856 5.023c1.179 1.962 1.379 5.119-1.6 8.098m-13.29-.528l-5.02-5.02a1 1 0 0 1 .707-1.701c.255 0 .512.098.707.292l2.607 2.607a.442.442 0 0 0 .624-.624L6.11 15.04l-1.75-1.75a.998.998 0 1 1 1.41-1.413l4.154 4.156a.44.44 0 0 0 .624 0 .44.44 0 0 0 0-.624l-4.152-4.153-1.172-1.171a.998.998 0 0 1 0-1.41 1.018 1.018 0 0 1 1.41 0l1.172 1.17 4.153 4.152a.437.437 0 0 0 .624 0 .442.442 0 0 0 0-.624L8.43 9.222a.988.988 0 0 1-.291-.705.99.99 0 0 1 .29-.706 1 1 0 0 1 1.412 0l6.992 6.993a.443.443 0 0 0 .71-.501l-1.35-2.856c-.315-.676-.235-1.185.246-1.557a.85.85 0 0 1 .66-.16c.342.056.659.28.879.606L20.628 15c1.573 2.876 1.067 5.545-1.544 8.156-1.396 1.397-3.144 1.966-5.063 1.652-1.713-.286-3.463-1.248-4.928-2.714zM12.99 6.976l2.562 2.562c-.497.607-.563 1.414-.155 2.284l.265.562-4.257-4.257a.98.98 0 0 1-.117-.445c0-.267.104-.517.292-.706a1.023 1.023 0 0 1 1.41 0zm8.887 2.06c-.375-.557-.902-.916-1.486-1.011a1.738 1.738 0 0 0-1.342.332c-.376.29-.61.656-.712 1.065a2.1 2.1 0 0 0-1.095-.562 1.776 1.776 0 0 0-.992.128l-2.636-2.636a1.883 1.883 0 0 0-2.658 0 1.862 1.862 0 0 0-.478.847 1.886 1.886 0 0 0-2.671-.012 1.867 1.867 0 0 0-.503.909c-.754-.754-1.992-.754-2.703-.044a1.881 1.881 0 0 0 0 2.658c-.288.12-.605.288-.864.547a1.884 1.884 0 0 0 0 2.659l.624.622a1.879 1.879 0 0 0-.91 3.16l5.019 5.02c1.595 1.594 3.515 2.645 5.408 2.959a7.16 7.16 0 0 0 1.173.098c1.026 0 1.997-.24 2.892-.7.279.04.555.065.828.065 1.53 0 2.969-.628 4.236-1.894 3.338-3.338 3.083-6.928 1.738-9.166l-2.868-5.043z"></path></g></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--clapFilled svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><g fill-rule="evenodd"><path d="M13.738 1l.762 2.966L15.262 1z"></path><path d="M18.634 2.224l-1.432-.47-.408 3.022z"></path><path d="M11.79 1.754l-1.431.47 1.84 2.552z"></path><path d="M24.472 14.307l-3.023-5.32c-.287-.426-.689-.705-1.123-.776a1.16 1.16 0 0 0-.911.221c-.297.231-.474.515-.535.84.017.022.036.04.053.063l2.843 5.001c1.95 3.564 1.328 6.973-1.843 10.144a8.46 8.46 0 0 1-.549.501c1.205-.156 2.328-.737 3.351-1.76 3.268-3.268 3.041-6.749 1.737-8.914"></path><path d="M14.58 10.887c-.156-.83.096-1.569.692-2.142L12.78 6.252c-.5-.504-1.378-.504-1.879 0-.178.18-.273.4-.329.63l4.008 4.005z"></path><path d="M17.812 10.04c-.218-.323-.539-.55-.88-.606a.814.814 0 0 0-.644.153c-.176.137-.713.553-.24 1.566l1.43 3.025a.539.539 0 1 1-.868.612L9.2 7.378a.986.986 0 1 0-1.395 1.395l4.401 4.403a.538.538 0 1 1-.762.762L7.046 9.54 5.802 8.295a.99.99 0 0 0-1.396 0 .981.981 0 0 0 0 1.394l1.241 1.241 4.402 4.403a.537.537 0 0 1 0 .761.535.535 0 0 1-.762 0L4.89 11.696a.992.992 0 0 0-1.399-.003.983.983 0 0 0 0 1.395l1.855 1.854 2.763 2.765a.538.538 0 0 1-.76.761l-2.765-2.764a.982.982 0 0 0-1.395 0 .989.989 0 0 0 0 1.395l5.32 5.32c3.371 3.372 6.64 4.977 10.49 1.126C21.74 20.8 22.271 18 20.62 14.982l-2.809-4.942z"></path></g></svg></span></span></button></div><span class="u-relative u-background js-actionMultirecommendCount u-marginLeft5"><button class="button button--chromeless u-baseColor--buttonNormal js-multirecommendCountButton" data-action="show-recommends" data-action-value="4ebd4711d0ec">1.93K</button></span></div></li><li class="u-marginVertical10 u-marginLeft3"><button class="button button--large button--dark button--chromeless is-touchIconFadeInPulse u-baseColor--buttonDark button--withIcon button--withSvgIcon button--bookmark js-bookmarkButton" title="Bookmark this story to read later" aria-label="Bookmark this story to read later" data-action="add-to-bookmarks" data-action-value="4ebd4711d0ec" data-action-source="post_share_widget-----4ebd4711d0ec---------------------bookmark_sidebar"><span class="button-defaultState"><span class="svgIcon svgIcon--bookmark svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M19.385 4h-9.77A2.623 2.623 0 0 0 7 6.615V23.01a1.022 1.022 0 0 0 1.595.847l5.905-4.004 5.905 4.004A1.022 1.022 0 0 0 22 23.011V6.62A2.625 2.625 0 0 0 19.385 4zM21 23l-5.91-3.955-.148-.107a.751.751 0 0 0-.884 0l-.147.107L8 23V6.615C8 5.725 8.725 5 9.615 5h9.77C20.275 5 21 5.725 21 6.615V23z" fill-rule="evenodd"></path></svg></span></span><span class="button-activeState"><span class="svgIcon svgIcon--bookmarkFilled svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M19.385 4h-9.77A2.623 2.623 0 0 0 7 6.615V23.01a1.022 1.022 0 0 0 1.595.847l5.905-4.004 5.905 4.004A1.022 1.022 0 0 0 22 23.011V6.62A2.625 2.625 0 0 0 19.385 4z" fill-rule="evenodd"></path></svg></span></span></button></li><li class="u-marginVertical10 u-marginLeft3"><a class="button button--dark button--chromeless u-baseColor--buttonDark button--withIcon button--withSvgIcon button--dark button--chromeless" href="https://medium.com/p/4ebd4711d0ec/share/twitter" title="Share on Twitter" aria-label="Share on Twitter" target="_blank" data-action-source="post_share_widget"><span class="button-defaultState"><span class="svgIcon svgIcon--twitterFilled svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M22.053 7.54a4.474 4.474 0 0 0-3.31-1.455 4.526 4.526 0 0 0-4.526 4.524c0 .35.04.7.082 1.05a12.9 12.9 0 0 1-9.3-4.77c-.39.69-.61 1.46-.65 2.26.03 1.6.83 2.99 2.02 3.79-.72-.02-1.41-.22-2.02-.57-.01.02-.01.04 0 .08-.01 2.17 1.55 4 3.63 4.44-.39.08-.79.13-1.21.16-.28-.03-.57-.05-.81-.08.54 1.77 2.21 3.08 4.2 3.15a9.564 9.564 0 0 1-5.66 1.94c-.34-.03-.7-.06-1.05-.08 2 1.27 4.38 2.02 6.94 2.02 8.31 0 12.86-6.9 12.84-12.85.02-.24.01-.43 0-.65.89-.62 1.65-1.42 2.26-2.34-.82.38-1.69.62-2.59.72a4.37 4.37 0 0 0 1.94-2.51c-.84.53-1.81.9-2.83 1.13z"></path></svg></span></span></a></li><li class="u-marginVertical10 u-marginLeft3"><a class="button button--dark button--chromeless u-baseColor--buttonDark button--withIcon button--withSvgIcon button--dark button--chromeless" href="https://medium.com/p/4ebd4711d0ec/share/facebook" title="Share on Facebook" aria-label="Share on Facebook" target="_blank" data-action-source="post_share_widget"><span class="button-defaultState"><span class="svgIcon svgIcon--facebookSquare svgIcon--29px"><svg class="svgIcon-use" width="29" height="29"><path d="M23.209 5H5.792A.792.792 0 0 0 5 5.791V23.21c0 .437.354.791.792.791h9.303v-7.125H12.72v-2.968h2.375v-2.375c0-2.455 1.553-3.662 3.741-3.662 1.049 0 1.95.078 2.213.112v2.565h-1.517c-1.192 0-1.469.567-1.469 1.397v1.963h2.969l-.594 2.968h-2.375L18.11 24h5.099a.791.791 0 0 0 .791-.791V5.79a.791.791 0 0 0-.791-.79"></path></svg></span></span></a></li></ul></div></aside><style class="js-collectionStyle">
.u-accentColor--borderLight {border-color: #668AAA !important;}
.u-accentColor--borderNormal {border-color: #668AAA !important;}
.u-accentColor--borderDark {border-color: #5A7690 !important;}
.u-accentColor--iconLight .svgIcon,.u-accentColor--iconLight.svgIcon {fill: #668AAA !important;}
.u-accentColor--iconNormal .svgIcon,.u-accentColor--iconNormal.svgIcon {fill: #668AAA !important;}
.u-accentColor--iconDark .svgIcon,.u-accentColor--iconDark.svgIcon {fill: #5A7690 !important;}
.u-accentColor--textNormal {color: #5A7690 !important;}
.u-accentColor--hoverTextNormal:hover {color: #5A7690 !important;}
.u-accentColor--textNormal.u-accentColor--textDarken:hover {color: #546C83 !important;}
.u-accentColor--textDark {color: #546C83 !important;}
.u-accentColor--backgroundLight {background-color: #668AAA !important;}
.u-accentColor--backgroundNormal {background-color: #668AAA !important;}
.u-accentColor--backgroundDark {background-color: #5A7690 !important;}
.u-accentColor--buttonDark {border-color: #5A7690 !important; color: #546C83 !important;}
.u-accentColor--buttonDark:hover {border-color: #546C83 !important;}
.u-accentColor--buttonDark .icon:before,.u-accentColor--buttonDark .svgIcon{color: #5A7690 !important; fill: #5A7690 !important;}
.u-accentColor--buttonNormal:not(.clapButton--largePill) {border-color: #668AAA !important; color: #5A7690 !important;}
.u-accentColor--buttonNormal:hover {border-color: #5A7690 !important;}
.u-accentColor--buttonNormal .icon:before,.u-accentColor--buttonNormal .svgIcon{color: #668AAA !important; fill: #668AAA !important;}
.u-accentColor--buttonNormal.button--filled .icon:before,.u-accentColor--buttonNormal.button--filled .svgIcon{color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-accentColor--buttonDark.button--filled,.u-accentColor--buttonDark.button--withChrome.is-active,.u-accentColor--fillWhenActive.is-active {background-color: #5A7690 !important; border-color: #5A7690 !important; color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-accentColor--buttonNormal.button--filled:not(.clapButton--largePill),.u-accentColor--buttonNormal.button--withChrome.is-active:not(.clapButton--largePill) {background-color: #668AAA !important; border-color: #668AAA !important; color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.postArticle.is-withAccentColors .markup--user,.postArticle.is-withAccentColors .markup--query {color: #5A7690 !important;}.u-tintBgColor {background-color: rgba(53, 88, 118, 1) !important;}.u-tintBgColor .u-fadeLeft:before {background-image: linear-gradient(to right, rgba(53, 88, 118, 1) 0%, rgba(53, 88, 118, 0) 100%) !important;}.u-tintBgColor .u-fadeRight:after {background-image: linear-gradient(to right, rgba(53, 88, 118, 0) 0%, rgba(53, 88, 118, 1) 100%) !important;}
.u-tintSpectrum .u-baseColor--borderLight {border-color: #9FB3C6 !important;}
.u-tintSpectrum .u-baseColor--borderNormal {border-color: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--borderDark {border-color: #E9F1FA !important;}
.u-tintSpectrum .u-baseColor--iconLight .svgIcon,.u-tintSpectrum .u-baseColor--iconLight.svgIcon {fill: #9FB3C6 !important;}
.u-tintSpectrum .u-baseColor--iconNormal .svgIcon,.u-tintSpectrum .u-baseColor--iconNormal.svgIcon {fill: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--iconDark .svgIcon,.u-tintSpectrum .u-baseColor--iconDark.svgIcon {fill: #E9F1FA !important;}
.u-tintSpectrum .u-baseColor--textNormal {color: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--textNormal.u-baseColor--textDarken:hover {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--textDark {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--textDarker {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--backgroundLight {background-color: #9FB3C6 !important;}
.u-tintSpectrum .u-baseColor--backgroundNormal {background-color: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--backgroundDark {background-color: #E9F1FA !important;}
.u-tintSpectrum .u-baseColor--buttonLight {border-color: #9FB3C6 !important; color: #9FB3C6 !important;}
.u-tintSpectrum .u-baseColor--buttonLight:hover {border-color: #9FB3C6 !important;}
.u-tintSpectrum .u-baseColor--buttonLight .icon:before,.u-tintSpectrum .u-baseColor--buttonLight .svgIcon {color: #9FB3C6 !important; fill: #9FB3C6 !important;}
.u-tintSpectrum .u-baseColor--buttonDark {border-color: #E9F1FA !important; color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--buttonDark:hover {border-color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--buttonDark .icon:before,.u-tintSpectrum .u-baseColor--buttonDark .svgIcon {color: #E9F1FA !important; fill: #E9F1FA !important;}
.u-tintSpectrum .u-baseColor--buttonNormal {border-color: #C5D2E1 !important; color: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--buttonNormal:hover {border-color: #E9F1FA !important;}
.u-tintSpectrum .u-baseColor--buttonNormal .icon:before,.u-tintSpectrum .u-baseColor--buttonNormal .svgIcon {color: #C5D2E1 !important; fill: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--buttonDark.button--filled,.u-tintSpectrum .u-baseColor--buttonDark.button--withChrome.is-active {background-color: #E9F1FA !important; border-color: #E9F1FA !important; color: rgba(53, 88, 118, 1) !important; fill: rgba(53, 88, 118, 1) !important;}
.u-tintSpectrum .u-baseColor--buttonNormal.button--filled,.u-tintSpectrum .u-baseColor--buttonNormal.button--withChrome.is-active {background-color: #C5D2E1 !important; border-color: #C5D2E1 !important; color: rgba(53, 88, 118, 1) !important; fill: rgba(53, 88, 118, 1) !important;}
.u-tintSpectrum .u-baseColor--link {color: #C5D2E1 !important;}
.u-tintSpectrum .u-baseColor--link.link--darkenOnHover:hover {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--link.link--darken:hover,.u-tintSpectrum .u-baseColor--link.link--darken:focus,.u-tintSpectrum .u-baseColor--link.link--darken:active {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--link.link--dark {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--link.link--dark.link--darken:hover,.u-tintSpectrum .u-baseColor--link.link--dark.link--darken:focus,.u-tintSpectrum .u-baseColor--link.link--dark.link--darken:active {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--link.link--darker {color: #FBFFFF !important;}
.u-tintSpectrum .u-baseColor--placeholderNormal ::-webkit-input-placeholder {color: #9FB3C6;}
.u-tintSpectrum .u-baseColor--placeholderNormal ::-moz-placeholder {color: #9FB3C6;}
.u-tintSpectrum .u-baseColor--placeholderNormal :-ms-input-placeholder {color: #9FB3C6;}
.u-tintSpectrum .svgIcon--logoWordmark {stroke: none !important; fill: #FBFFFF !important;}
.u-tintSpectrum .svgIcon--logoMonogram {stroke: none !important; fill: #FBFFFF !important;}
.u-tintSpectrum  .ui-h1,.u-tintSpectrum  .ui-h2,.u-tintSpectrum  .ui-h3,.u-tintSpectrum  .ui-h4,.u-tintSpectrum  .ui-brand1,.u-tintSpectrum  .ui-brand2,.u-tintSpectrum  .ui-captionStrong {color: #FBFFFF !important; fill: #FBFFFF !important;}
.u-tintSpectrum  .ui-body,.u-tintSpectrum  .ui-caps {color: #FBFFFF !important; fill: #FBFFFF !important;}
.u-tintSpectrum  .ui-summary,.u-tintSpectrum  .ui-caption {color: #9FB3C6 !important; fill: #9FB3C6 !important;}
.u-tintSpectrum .u-accentColor--borderLight {border-color: #9FB3C6 !important;}
.u-tintSpectrum .u-accentColor--borderNormal {border-color: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--borderDark {border-color: #E9F1FA !important;}
.u-tintSpectrum .u-accentColor--iconLight .svgIcon,.u-tintSpectrum .u-accentColor--iconLight.svgIcon {fill: #9FB3C6 !important;}
.u-tintSpectrum .u-accentColor--iconNormal .svgIcon,.u-tintSpectrum .u-accentColor--iconNormal.svgIcon {fill: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--iconDark .svgIcon,.u-tintSpectrum .u-accentColor--iconDark.svgIcon {fill: #E9F1FA !important;}
.u-tintSpectrum .u-accentColor--textNormal {color: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--hoverTextNormal:hover {color: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--textNormal.u-accentColor--textDarken:hover {color: #FBFFFF !important;}
.u-tintSpectrum .u-accentColor--textDark {color: #FBFFFF !important;}
.u-tintSpectrum .u-accentColor--backgroundLight {background-color: #9FB3C6 !important;}
.u-tintSpectrum .u-accentColor--backgroundNormal {background-color: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--backgroundDark {background-color: #E9F1FA !important;}
.u-tintSpectrum .u-accentColor--buttonDark {border-color: #E9F1FA !important; color: #FBFFFF !important;}
.u-tintSpectrum .u-accentColor--buttonDark:hover {border-color: #FBFFFF !important;}
.u-tintSpectrum .u-accentColor--buttonDark .icon:before,.u-tintSpectrum .u-accentColor--buttonDark .svgIcon{color: #E9F1FA !important; fill: #E9F1FA !important;}
.u-tintSpectrum .u-accentColor--buttonNormal:not(.clapButton--largePill) {border-color: #C5D2E1 !important; color: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--buttonNormal:hover {border-color: #E9F1FA !important;}
.u-tintSpectrum .u-accentColor--buttonNormal .icon:before,.u-tintSpectrum .u-accentColor--buttonNormal .svgIcon{color: #C5D2E1 !important; fill: #C5D2E1 !important;}
.u-tintSpectrum .u-accentColor--buttonNormal.button--filled .icon:before,.u-tintSpectrum .u-accentColor--buttonNormal.button--filled .svgIcon{color: rgba(53, 88, 118, 1) !important; fill: rgba(53, 88, 118, 1) !important;}
.u-tintSpectrum .u-accentColor--buttonDark.button--filled,.u-tintSpectrum .u-accentColor--buttonDark.button--withChrome.is-active,.u-tintSpectrum .u-accentColor--fillWhenActive.is-active {background-color: #E9F1FA !important; border-color: #E9F1FA !important; color: rgba(53, 88, 118, 1) !important; fill: rgba(53, 88, 118, 1) !important;}
.u-tintSpectrum .u-accentColor--buttonNormal.button--filled:not(.clapButton--largePill),.u-tintSpectrum .u-accentColor--buttonNormal.button--withChrome.is-active:not(.clapButton--largePill) {background-color: #C5D2E1 !important; border-color: #C5D2E1 !important; color: rgba(53, 88, 118, 1) !important; fill: rgba(53, 88, 118, 1) !important;}
.u-tintSpectrum .postArticle.is-withAccentColors .markup--user,.u-tintSpectrum .postArticle.is-withAccentColors .markup--query {color: #C5D2E1 !important;}
.u-accentColor--highlightFaint {background-color: rgba(233, 242, 253, 1) !important;}
.u-accentColor--highlightStrong.is-active .svgIcon {fill: rgba(200, 228, 255, 1) !important;}
.postArticle.is-withAccentColors .markup--quote.is-other {background-color: rgba(233, 242, 253, 1) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .markup--quote.is-other {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(233, 242, 253, 1), rgba(233, 242, 253, 1));}
.postArticle.is-withAccentColors .markup--quote.is-me {background-color: rgba(215, 235, 254, 1) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .markup--quote.is-me {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(215, 235, 254, 1), rgba(215, 235, 254, 1));}
.postArticle.is-withAccentColors .markup--quote.is-targeted {background-color: rgba(200, 228, 255, 1) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .markup--quote.is-targeted {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(200, 228, 255, 1), rgba(200, 228, 255, 1));}
.postArticle.is-withAccentColors .markup--quote.is-selected {background-color: rgba(200, 228, 255, 1) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .markup--quote.is-selected {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(200, 228, 255, 1), rgba(200, 228, 255, 1));}
.postArticle.is-withAccentColors .markup--highlight {background-color: rgba(200, 228, 255, 1) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .markup--highlight {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(200, 228, 255, 1), rgba(200, 228, 255, 1));}.u-baseColor--iconNormal.avatar-halo {fill: rgba(0, 0, 0, 0.4980392156862745) !important;}</style><style class="js-collectionStyleConstant">.u-imageBgColor {background-color: rgba(0, 0, 0, 0.24705882352941178);}
.u-imageSpectrum .u-baseColor--borderLight {border-color: rgba(255, 255, 255, 0.6980392156862745) !important;}
.u-imageSpectrum .u-baseColor--borderNormal {border-color: rgba(255, 255, 255, 0.8980392156862745) !important;}
.u-imageSpectrum .u-baseColor--borderDark {border-color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--iconLight .svgIcon,.u-imageSpectrum .u-baseColor--iconLight.svgIcon {fill: rgba(255, 255, 255, 0.8) !important;}
.u-imageSpectrum .u-baseColor--iconNormal .svgIcon,.u-imageSpectrum .u-baseColor--iconNormal.svgIcon {fill: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--iconDark .svgIcon,.u-imageSpectrum .u-baseColor--iconDark.svgIcon {fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--textNormal {color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--textNormal.u-baseColor--textDarken:hover {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--textDark {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--textDarker {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--backgroundLight {background-color: rgba(255, 255, 255, 0.8980392156862745) !important;}
.u-imageSpectrum .u-baseColor--backgroundNormal {background-color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--backgroundDark {background-color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--buttonLight {border-color: rgba(255, 255, 255, 0.6980392156862745) !important; color: rgba(255, 255, 255, 0.8) !important;}
.u-imageSpectrum .u-baseColor--buttonLight:hover {border-color: rgba(255, 255, 255, 0.6980392156862745) !important;}
.u-imageSpectrum .u-baseColor--buttonLight .icon:before,.u-imageSpectrum .u-baseColor--buttonLight .svgIcon {color: rgba(255, 255, 255, 0.8) !important; fill: rgba(255, 255, 255, 0.8) !important;}
.u-imageSpectrum .u-baseColor--buttonDark {border-color: rgba(255, 255, 255, 0.9490196078431372) !important; color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--buttonDark:hover {border-color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--buttonDark .icon:before,.u-imageSpectrum .u-baseColor--buttonDark .svgIcon {color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--buttonNormal {border-color: rgba(255, 255, 255, 0.8980392156862745) !important; color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--buttonNormal:hover {border-color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--buttonNormal .icon:before,.u-imageSpectrum .u-baseColor--buttonNormal .svgIcon {color: rgba(255, 255, 255, 0.9490196078431372) !important; fill: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--buttonDark.button--filled,.u-imageSpectrum .u-baseColor--buttonDark.button--withChrome.is-active {background-color: rgba(255, 255, 255, 1) !important; border-color: rgba(255, 255, 255, 1) !important; color: rgba(0, 0, 0, 0.24705882352941178) !important; fill: rgba(0, 0, 0, 0.24705882352941178) !important;}
.u-imageSpectrum .u-baseColor--buttonNormal.button--filled,.u-imageSpectrum .u-baseColor--buttonNormal.button--withChrome.is-active {background-color: rgba(255, 255, 255, 0.9490196078431372) !important; border-color: rgba(255, 255, 255, 0.9490196078431372) !important; color: rgba(0, 0, 0, 0.24705882352941178) !important; fill: rgba(0, 0, 0, 0.24705882352941178) !important;}
.u-imageSpectrum .u-baseColor--link {color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-baseColor--link.link--darkenOnHover:hover {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--link.link--darken:hover,.u-imageSpectrum .u-baseColor--link.link--darken:focus,.u-imageSpectrum .u-baseColor--link.link--darken:active {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--link.link--dark {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--link.link--dark.link--darken:hover,.u-imageSpectrum .u-baseColor--link.link--dark.link--darken:focus,.u-imageSpectrum .u-baseColor--link.link--dark.link--darken:active {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--link.link--darker {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-baseColor--placeholderNormal ::-webkit-input-placeholder {color: rgba(255, 255, 255, 0.8);}
.u-imageSpectrum .u-baseColor--placeholderNormal ::-moz-placeholder {color: rgba(255, 255, 255, 0.8);}
.u-imageSpectrum .u-baseColor--placeholderNormal :-ms-input-placeholder {color: rgba(255, 255, 255, 0.8);}
.u-imageSpectrum .svgIcon--logoWordmark {stroke: none !important; fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .svgIcon--logoMonogram {stroke: none !important; fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum  .ui-h1,.u-imageSpectrum  .ui-h2,.u-imageSpectrum  .ui-h3,.u-imageSpectrum  .ui-h4,.u-imageSpectrum  .ui-brand1,.u-imageSpectrum  .ui-brand2,.u-imageSpectrum  .ui-captionStrong {color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum  .ui-body,.u-imageSpectrum  .ui-caps {color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum  .ui-summary,.u-imageSpectrum  .ui-caption {color: rgba(255, 255, 255, 0.8) !important; fill: rgba(255, 255, 255, 0.8) !important;}
.u-imageSpectrum .u-accentColor--borderLight {border-color: rgba(255, 255, 255, 0.6980392156862745) !important;}
.u-imageSpectrum .u-accentColor--borderNormal {border-color: rgba(255, 255, 255, 0.8980392156862745) !important;}
.u-imageSpectrum .u-accentColor--borderDark {border-color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--iconLight .svgIcon,.u-imageSpectrum .u-accentColor--iconLight.svgIcon {fill: rgba(255, 255, 255, 0.8) !important;}
.u-imageSpectrum .u-accentColor--iconNormal .svgIcon,.u-imageSpectrum .u-accentColor--iconNormal.svgIcon {fill: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--iconDark .svgIcon,.u-imageSpectrum .u-accentColor--iconDark.svgIcon {fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--textNormal {color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--hoverTextNormal:hover {color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--textNormal.u-accentColor--textDarken:hover {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--textDark {color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--backgroundLight {background-color: rgba(255, 255, 255, 0.8980392156862745) !important;}
.u-imageSpectrum .u-accentColor--backgroundNormal {background-color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--backgroundDark {background-color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--buttonDark {border-color: rgba(255, 255, 255, 0.9490196078431372) !important; color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--buttonDark:hover {border-color: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--buttonDark .icon:before,.u-imageSpectrum .u-accentColor--buttonDark .svgIcon{color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-imageSpectrum .u-accentColor--buttonNormal:not(.clapButton--largePill) {border-color: rgba(255, 255, 255, 0.8980392156862745) !important; color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--buttonNormal:hover {border-color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--buttonNormal .icon:before,.u-imageSpectrum .u-accentColor--buttonNormal .svgIcon{color: rgba(255, 255, 255, 0.9490196078431372) !important; fill: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--buttonNormal.button--filled .icon:before,.u-imageSpectrum .u-accentColor--buttonNormal.button--filled .svgIcon{color: rgba(0, 0, 0, 0.24705882352941178) !important; fill: rgba(0, 0, 0, 0.24705882352941178) !important;}
.u-imageSpectrum .u-accentColor--buttonDark.button--filled,.u-imageSpectrum .u-accentColor--buttonDark.button--withChrome.is-active,.u-imageSpectrum .u-accentColor--fillWhenActive.is-active {background-color: rgba(255, 255, 255, 1) !important; border-color: rgba(255, 255, 255, 1) !important; color: rgba(0, 0, 0, 0.24705882352941178) !important; fill: rgba(0, 0, 0, 0.24705882352941178) !important;}
.u-imageSpectrum .u-accentColor--buttonNormal.button--filled:not(.clapButton--largePill),.u-imageSpectrum .u-accentColor--buttonNormal.button--withChrome.is-active:not(.clapButton--largePill) {background-color: rgba(255, 255, 255, 0.9490196078431372) !important; border-color: rgba(255, 255, 255, 0.9490196078431372) !important; color: rgba(0, 0, 0, 0.24705882352941178) !important; fill: rgba(0, 0, 0, 0.24705882352941178) !important;}
.u-imageSpectrum .postArticle.is-withAccentColors .markup--user,.u-imageSpectrum .postArticle.is-withAccentColors .markup--query {color: rgba(255, 255, 255, 0.9490196078431372) !important;}
.u-imageSpectrum .u-accentColor--highlightFaint {background-color: rgba(255, 255, 255, 0.2) !important;}
.u-imageSpectrum .u-accentColor--highlightStrong.is-active .svgIcon {fill: rgba(255, 255, 255, 0.6) !important;}
.postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-other {background-color: rgba(255, 255, 255, 0.2) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-other {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 0.2));}
.postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-me {background-color: rgba(255, 255, 255, 0.4) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-me {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(255, 255, 255, 0.4), rgba(255, 255, 255, 0.4));}
.postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-targeted {background-color: rgba(255, 255, 255, 0.6) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-targeted {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(255, 255, 255, 0.6), rgba(255, 255, 255, 0.6));}
.postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-selected {background-color: rgba(255, 255, 255, 0.6) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .u-imageSpectrum .markup--quote.is-selected {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(255, 255, 255, 0.6), rgba(255, 255, 255, 0.6));}
.postArticle.is-withAccentColors .u-imageSpectrum .markup--highlight {background-color: rgba(255, 255, 255, 0.6) !important;}
body.is-withMagicUnderlines .postArticle.is-withAccentColors .u-imageSpectrum .markup--highlight {background-color: transparent !important; background-image: linear-gradient(to bottom, rgba(255, 255, 255, 0.6), rgba(255, 255, 255, 0.6));}.u-resetSpectrum .u-tintBgColor {background-color: rgba(255, 255, 255, 1) !important;}.u-resetSpectrum .u-tintBgColor .u-fadeLeft:before {background-image: linear-gradient(to right, rgba(255, 255, 255, 1) 0%, rgba(255, 255, 255, 0) 100%) !important;}.u-resetSpectrum .u-tintBgColor .u-fadeRight:after {background-image: linear-gradient(to right, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 1) 100%) !important;}
.u-resetSpectrum .u-baseColor--borderLight {border-color: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-baseColor--borderNormal {border-color: rgba(0, 0, 0, 0.4980392156862745) !important;}
.u-resetSpectrum .u-baseColor--borderDark {border-color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--iconLight .svgIcon,.u-resetSpectrum .u-baseColor--iconLight.svgIcon {fill: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-baseColor--iconNormal .svgIcon,.u-resetSpectrum .u-baseColor--iconNormal.svgIcon {fill: rgba(0, 0, 0, 0.4980392156862745) !important;}
.u-resetSpectrum .u-baseColor--iconDark .svgIcon,.u-resetSpectrum .u-baseColor--iconDark.svgIcon {fill: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--textNormal {color: rgba(0, 0, 0, 0.4980392156862745) !important;}
.u-resetSpectrum .u-baseColor--textNormal.u-baseColor--textDarken:hover {color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--textDark {color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--textDarker {color: rgba(0, 0, 0, 0.8) !important;}
.u-resetSpectrum .u-baseColor--backgroundLight {background-color: rgba(0, 0, 0, 0.09803921568627451) !important;}
.u-resetSpectrum .u-baseColor--backgroundNormal {background-color: rgba(0, 0, 0, 0.2) !important;}
.u-resetSpectrum .u-baseColor--backgroundDark {background-color: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-baseColor--buttonLight {border-color: rgba(0, 0, 0, 0.2980392156862745) !important; color: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-baseColor--buttonLight:hover {border-color: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-baseColor--buttonLight .icon:before,.u-resetSpectrum .u-baseColor--buttonLight .svgIcon {color: rgba(0, 0, 0, 0.2980392156862745) !important; fill: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-baseColor--buttonDark {border-color: rgba(0, 0, 0, 0.6) !important; color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--buttonDark:hover {border-color: rgba(0, 0, 0, 0.8) !important;}
.u-resetSpectrum .u-baseColor--buttonDark .icon:before,.u-resetSpectrum .u-baseColor--buttonDark .svgIcon {color: rgba(0, 0, 0, 0.6) !important; fill: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--buttonNormal {border-color: rgba(0, 0, 0, 0.4980392156862745) !important; color: rgba(0, 0, 0, 0.4980392156862745) !important;}
.u-resetSpectrum .u-baseColor--buttonNormal:hover {border-color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--buttonNormal .icon:before,.u-resetSpectrum .u-baseColor--buttonNormal .svgIcon {color: rgba(0, 0, 0, 0.4980392156862745) !important; fill: rgba(0, 0, 0, 0.4980392156862745) !important;}
.u-resetSpectrum .u-baseColor--buttonDark.button--filled,.u-resetSpectrum .u-baseColor--buttonDark.button--withChrome.is-active {background-color: rgba(0, 0, 0, 0.2980392156862745) !important; border-color: rgba(0, 0, 0, 0.2980392156862745) !important; color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-resetSpectrum .u-baseColor--buttonNormal.button--filled,.u-resetSpectrum .u-baseColor--buttonNormal.button--withChrome.is-active {background-color: rgba(0, 0, 0, 0.2) !important; border-color: rgba(0, 0, 0, 0.2) !important; color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-resetSpectrum .u-baseColor--link {color: rgba(0, 0, 0, 0.4980392156862745) !important;}
.u-resetSpectrum .u-baseColor--link.link--darkenOnHover:hover {color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--link.link--darken:hover,.u-resetSpectrum .u-baseColor--link.link--darken:focus,.u-resetSpectrum .u-baseColor--link.link--darken:active {color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--link.link--dark {color: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .u-baseColor--link.link--dark.link--darken:hover,.u-resetSpectrum .u-baseColor--link.link--dark.link--darken:focus,.u-resetSpectrum .u-baseColor--link.link--dark.link--darken:active {color: rgba(0, 0, 0, 0.8) !important;}
.u-resetSpectrum .u-baseColor--link.link--darker {color: rgba(0, 0, 0, 0.8) !important;}
.u-resetSpectrum .u-baseColor--placeholderNormal ::-webkit-input-placeholder {color: rgba(0, 0, 0, 0.2980392156862745);}
.u-resetSpectrum .u-baseColor--placeholderNormal ::-moz-placeholder {color: rgba(0, 0, 0, 0.2980392156862745);}
.u-resetSpectrum .u-baseColor--placeholderNormal :-ms-input-placeholder {color: rgba(0, 0, 0, 0.2980392156862745);}
.u-resetSpectrum .svgIcon--logoWordmark {stroke: none !important; fill: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum .svgIcon--logoMonogram {stroke: none !important; fill: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum  .ui-h1,.u-resetSpectrum  .ui-h2,.u-resetSpectrum  .ui-h3,.u-resetSpectrum  .ui-h4,.u-resetSpectrum  .ui-brand1,.u-resetSpectrum  .ui-brand2,.u-resetSpectrum  .ui-captionStrong {color: rgba(0, 0, 0, 0.8) !important; fill: rgba(0, 0, 0, 0.8) !important;}
.u-resetSpectrum  .ui-body,.u-resetSpectrum  .ui-caps {color: rgba(0, 0, 0, 0.6) !important; fill: rgba(0, 0, 0, 0.6) !important;}
.u-resetSpectrum  .ui-summary,.u-resetSpectrum  .ui-caption {color: rgba(0, 0, 0, 0.2980392156862745) !important; fill: rgba(0, 0, 0, 0.2980392156862745) !important;}
.u-resetSpectrum .u-accentColor--borderLight {border-color: rgba(2, 184, 117, 1) !important;}
.u-resetSpectrum .u-accentColor--borderNormal {border-color: rgba(2, 184, 117, 1) !important;}
.u-resetSpectrum .u-accentColor--borderDark {border-color: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--iconLight .svgIcon,.u-resetSpectrum .u-accentColor--iconLight.svgIcon {fill: rgba(2, 184, 117, 1) !important;}
.u-resetSpectrum .u-accentColor--iconNormal .svgIcon,.u-resetSpectrum .u-accentColor--iconNormal.svgIcon {fill: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--iconDark .svgIcon,.u-resetSpectrum .u-accentColor--iconDark.svgIcon {fill: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--textNormal {color: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--hoverTextNormal:hover {color: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--textNormal.u-accentColor--textDarken:hover {color: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--textDark {color: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--backgroundLight {background-color: rgba(2, 184, 117, 1) !important;}
.u-resetSpectrum .u-accentColor--backgroundNormal {background-color: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--backgroundDark {background-color: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonDark {border-color: rgba(0, 171, 107, 1) !important; color: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonDark:hover {border-color: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonDark .icon:before,.u-resetSpectrum .u-accentColor--buttonDark .svgIcon{color: rgba(28, 153, 99, 1) !important; fill: rgba(28, 153, 99, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonNormal:not(.clapButton--largePill) {border-color: rgba(2, 184, 117, 1) !important; color: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonNormal:hover {border-color: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonNormal .icon:before,.u-resetSpectrum .u-accentColor--buttonNormal .svgIcon{color: rgba(0, 171, 107, 1) !important; fill: rgba(0, 171, 107, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonNormal.button--filled .icon:before,.u-resetSpectrum .u-accentColor--buttonNormal.button--filled .svgIcon{color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonDark.button--filled,.u-resetSpectrum .u-accentColor--buttonDark.button--withChrome.is-active,.u-resetSpectrum .u-accentColor--fillWhenActive.is-active {background-color: rgba(28, 153, 99, 1) !important; border-color: rgba(28, 153, 99, 1) !important; color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-resetSpectrum .u-accentColor--buttonNormal.button--filled:not(.clapButton--largePill),.u-resetSpectrum .u-accentColor--buttonNormal.button--withChrome.is-active:not(.clapButton--largePill) {background-color: rgba(0, 171, 107, 1) !important; border-color: rgba(0, 171, 107, 1) !important; color: rgba(255, 255, 255, 1) !important; fill: rgba(255, 255, 255, 1) !important;}
.u-resetSpectrum .postArticle.is-withAccentColors .markup--user,.u-resetSpectrum .postArticle.is-withAccentColors .markup--query {color: rgba(0, 171, 107, 1) !important;}</style><div class="highlightMenu" data-action-scope="_actionscope_3"><div class="highlightMenu-inner"><div class="buttonSet"><button class="button button--chromeless u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--highlightMenu u-accentColor--highlightStrong js-highlightMenuQuoteButton" data-action="quote" data-action-source="quote_menu--------------------------highlight_text" data-skip-onboarding="true"><span class="svgIcon svgIcon--highlighter svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M13.7 15.964l5.204-9.387-4.726-2.62-5.204 9.387 4.726 2.62zm-.493.885l-1.313 2.37-1.252.54-.702 1.263-3.796-.865 1.228-2.213-.202-1.35 1.314-2.37 4.722 2.616z" fill-rule="evenodd"></path></svg></span></button><button class="button button--chromeless u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--highlightMenu" data-action="quote-respond" data-action-source="quote_menu--------------------------respond_text" data-skip-onboarding="true"><span class="svgIcon svgIcon--responseFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M19.074 21.117c-1.244 0-2.432-.37-3.532-1.096a7.792 7.792 0 0 1-.703-.52c-.77.21-1.57.32-2.38.32-4.67 0-8.46-3.5-8.46-7.8C4 7.7 7.79 4.2 12.46 4.2c4.662 0 8.457 3.5 8.457 7.803 0 2.058-.85 3.984-2.403 5.448.023.17.06.35.118.55.192.69.537 1.38 1.026 2.04.15.21.172.48.058.7a.686.686 0 0 1-.613.38h-.03z" fill-rule="evenodd"></path></svg></span></button><a class="button button--chromeless u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--chromeless button--highlightMenu js-highlightMenuTwitterShare" href="https://medium.com/p/4ebd4711d0ec/share/twitter" title="Share on Twitter" aria-label="Share on Twitter" target="_blank" data-action="twitter"><span class="button-defaultState"><span class="svgIcon svgIcon--twitterFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><path d="M21.725 5.338c-.744.47-1.605.804-2.513 1.006a3.978 3.978 0 0 0-2.942-1.293c-2.22 0-4.02 1.81-4.02 4.02 0 .32.034.63.07.94-3.31-.18-6.27-1.78-8.255-4.23a4.544 4.544 0 0 0-.574 2.01c.04 1.43.74 2.66 1.8 3.38-.63-.01-1.25-.19-1.79-.5v.08c0 1.93 1.38 3.56 3.23 3.95-.34.07-.7.12-1.07.14-.25-.02-.5-.04-.72-.07.49 1.58 1.97 2.74 3.74 2.8a8.49 8.49 0 0 1-5.02 1.72c-.3-.03-.62-.04-.93-.07A11.447 11.447 0 0 0 8.88 21c7.386 0 11.43-6.13 11.414-11.414.015-.21.01-.38 0-.578a7.604 7.604 0 0 0 2.01-2.08 7.27 7.27 0 0 1-2.297.645 3.856 3.856 0 0 0 1.72-2.23"></path></svg></span></span></a><div class="buttonSet-separator"></div><button class="button button--chromeless u-baseColor--buttonNormal button--withIcon button--withSvgIcon button--highlightMenu" data-action="highlight" data-action-source="quote_menu--------------------------privatenote_text" data-skip-onboarding="true"><span class="svgIcon svgIcon--privatenoteFilled svgIcon--25px"><svg class="svgIcon-use" width="25" height="25"><g fill-rule="evenodd"><path d="M17.662 4.552H7.346A4.36 4.36 0 0 0 3 8.898v5.685c0 2.168 1.614 3.962 3.697 4.28v2.77c0 .303.35.476.59.29l3.904-2.994h6.48c2.39 0 4.35-1.96 4.35-4.35V8.9c0-2.39-1.95-4.346-4.34-4.346zM16 14.31a.99.99 0 0 1-1.003.99h-4.994C9.45 15.3 9 14.85 9 14.31v-3.02a.99.99 0 0 1 1-.99v-.782a2.5 2.5 0 0 1 2.5-2.51c1.38 0 2.5 1.13 2.5 2.51v.782c.552.002 1 .452 1 .99v3.02z"></path><path d="M14 9.81c0-.832-.674-1.68-1.5-1.68-.833 0-1.5.84-1.5 1.68v.49h3v-.49z"></path></g></svg></span></button></div></div><div class="highlightMenu-arrowClip"><span class="highlightMenu-arrow"></span></div></div></div></div></div><div class="loadingBar"></div><script>// <![CDATA[
window["obvInit"] = function (opt_embedded) {window["obvInit"]["embedded"] = opt_embedded; window["obvInit"]["ready"] = true;}
// ]]></script><script>// <![CDATA[
var GLOBALS = {"audioUrl":"https://d1fcbxp97j4nb2.cloudfront.net","baseUrl":"https://towardsdatascience.com","buildLabel":"37959-ef62632","currentUser":{"userId":"3751a3493996","username":"wangchj04","name":"Cheng-Jun Wang","email":"wangchj04@gmail.com","imageId":"0*Mpgm7EXaTEICpUOi.jpg","createdAt":1557477875037,"isVerified":true,"subscriberEmail":"","onboardingStatus":1,"googleAccountId":"108374965119894760200","googleEmail":"wangchj04@gmail.com","hasPastMemberships":false,"isEnrolledInHightower":false,"isEligibleForHightower":true,"hightowerLastLockedAt":0,"isWriterProgramEnrolled":true,"isWriterProgramInvited":true,"isWriterProgramOptedOut":false,"writerProgramVersion":5,"writerProgramEnrolledAt":1557477875037,"friendLinkOnboarding":0,"hasAdditionalUnlocks":false,"hasApiAccess":false,"isQuarantined":false,"writerProgramDistributionSettingOptedIn":true},"currentUserHasUnverifiedEmail":false,"isAuthenticated":true,"isCurrentUserVerified":true,"language":"zh-tw","miroUrl":"https://cdn-images-1.medium.com","moduleUrls":{"base":"https://cdn-static-1.medium.com/_/fp/gen-js/main-base.bundle.CwzueaBQKTFG7DV3x_5v9A.js","common-async":"https://cdn-static-1.medium.com/_/fp/gen-js/main-common-async.bundle.mBB8x4lNtOgIEyw4wVO22w.js","hightower":"https://cdn-static-1.medium.com/_/fp/gen-js/main-hightower.bundle.p1xyIHWAXenU68zbbTR95g.js","home-screens":"https://cdn-static-1.medium.com/_/fp/gen-js/main-home-screens.bundle.2H0uQtznBDlgve7K27M3YA.js","misc-screens":"https://cdn-static-1.medium.com/_/fp/gen-js/main-misc-screens.bundle.Eu30GGSjs22roEX6-iXv0w.js","notes":"https://cdn-static-1.medium.com/_/fp/gen-js/main-notes.bundle.m6hqQlEdhwrG8mHQcqEXww.js","payments":"https://cdn-static-1.medium.com/_/fp/gen-js/main-payments.bundle.E4giPnk1B_-y07fU7Mq-sg.js","posters":"https://cdn-static-1.medium.com/_/fp/gen-js/main-posters.bundle.1B9ZW8BQO9iQjfidV-fzzw.js","power-readers":"https://cdn-static-1.medium.com/_/fp/gen-js/main-power-readers.bundle.aXds_tYS2qk1wKYO-eE4vQ.js","pubs":"https://cdn-static-1.medium.com/_/fp/gen-js/main-pubs.bundle.cQmrwnFu11-89ZpFtgL4kw.js","stats":"https://cdn-static-1.medium.com/_/fp/gen-js/main-stats.bundle.VE9T3tLwsrzMGclGqSDAWw.js"},"previewConfig":{"weightThreshold":1,"weightImageParagraph":0.51,"weightIframeParagraph":0.8,"weightTextParagraph":0.08,"weightEmptyParagraph":0,"weightP":0.003,"weightH":0.005,"weightBq":0.003,"minPTextLength":60,"truncateBoundaryChars":20,"detectTitle":true,"detectTitleLevThreshold":0.15},"productName":"Medium","supportsEdit":true,"termsUrl":"//medium.com/policy/9db0094a1e0f","textshotHost":"textshot.medium.com","transactionId":"1561519658584:abc56f6b1b89","useragent":{"browser":"chrome","family":"chrome","os":"mac","version":74,"supportsDesktopEdit":true,"supportsInteract":true,"supportsView":true,"isMobile":false,"isTablet":false,"isNative":false,"supportsFileAPI":true,"isTier1":true,"clientVersion":"","unknownParagraphsBad":false,"clientChannel":"","supportsRealScrollEvents":true,"supportsVhUnits":true,"ruinsViewportSections":false,"supportsHtml5Video":true,"supportsMagicUnderlines":true,"isWebView":false,"isFacebookWebView":false,"supportsProgressiveMedia":true,"supportsPromotedPosts":true,"isBot":false,"isNativeIphone":false,"supportsCssVariables":true,"supportsVideoSections":true,"emojiSupportLevel":5,"isSearchBot":false,"isSyndicationBot":false,"isNativeAndroid":false,"isNativeIos":false,"isSeoBot":false,"supportsScrollableMetabar":true},"variants":{"allow_access":true,"allow_signup":true,"allow_test_auth":"disallow","signin_services":"twitter,facebook,google,email,google-fastidv,google-one-tap","signup_services":"twitter,facebook,google,email,google-fastidv,google-one-tap","google_sign_in_android":true,"reengagement_notification_duration":3,"browsable_stream_config_bucket":"curated-topics","enable_dedicated_series_tab_api_ios":true,"enable_post_import":true,"available_monthly_plan":"60e220181034","available_annual_plan":"2c754bcc2995","disable_ios_resume_reading_toast":true,"is_not_medium_subscriber":true,"glyph_font_set":"m2","enable_branding":true,"enable_branding_fonts":true,"max_premium_content_per_user_under_metering":3,"enable_automated_mission_control_triggers":true,"enable_lite_profile":true,"enable_marketing_emails":true,"enable_topic_lifecycle_email":true,"enable_parsely":true,"enable_branch_io":true,"enable_ios_post_stats":true,"enable_lite_topics":true,"enable_lite_stories":true,"redis_read_write_splitting":true,"enable_tipalti_onboarding":true,"enable_international_tax_withholding":true,"enable_international_tax_withholding_documentation":true,"enable_revised_first_partner_program_distro_on_email":true,"enable_annual_renewal_reminder_email":true,"enable_janky_spam_rules":"users,posts","enable_new_collaborative_filtering_data":true,"android_rating_prompt_stories_read_threshold":2,"enable_google_one_tap":true,"enable_email_sign_in_captcha":true,"enable_primary_topic_for_mobile":true,"enable_logged_out_homepage_signup":true,"use_new_admin_topic_backend":true,"enable_quarantine_rules":true,"enable_patronus_on_kubernetes":true,"pub_sidebar":true,"disable_mobile_featured_chunk":true,"enable_embedding_based_diversification":true,"enable_pub_newsletters":true,"enable_may_meter_email_test":true,"enable_pppp_pub_sidebar":true,"enable_draft_in_post_cotent":true,"enable_lite_claps":true,"enable_retrained_ranking_model_digest":true,"enable_retrained_ranking_model_homepage":true,"enable_lite_post_manager_gear_menu":true,"enable_live_user_post_scoring":true,"enable_pppp_report_story_modal":true,"enable_lite_meter_controller":true,"enable_lite_post_highlights_view_only":true,"enable_tick_landing_page":true,"enable_lite_private_notes":true,"enable_lite_email_sign_in_flow":true,"enable_daily_read_digest_promo":true,"enable_pride_logo":true,"enable_serve_recs_from_ml_rank_homepage":true,"enable_serve_recs_from_ml_rank_digest":true,"enable_serve_recs_from_ml_rank_app_highlights":true,"unsubscribe_from_medium_newsletters":true,"enable_lite_google_captcha":true},"xsrfToken":"I5PgatBUQzHhMpvP","iosAppId":"828256236","supportEmail":"yourfriends@medium.com","fp":{"/icons/monogram-mask.svg":"https://cdn-static-1.medium.com/_/fp/icons/monogram-mask.KPLCSFEZviQN0jQ7veN2RQ.svg","/icons/favicon-dev-editor.ico":"https://cdn-static-1.medium.com/_/fp/icons/favicon-dev-editor.YKKRxBO8EMvIqhyCwIiJeQ.ico","/icons/favicon-hatch-editor.ico":"https://cdn-static-1.medium.com/_/fp/icons/favicon-hatch-editor.BuEyHIqlyh2s_XEk4Rl32Q.ico","/icons/favicon-medium-editor.ico":"https://cdn-static-1.medium.com/_/fp/icons/favicon-medium-editor.PiakrZWB7Yb80quUVQWM6g.ico"},"authBaseUrl":"https://medium.com","imageUploadSizeMb":25,"isAuthDomainRequest":false,"domainCollectionSlug":"towards-data-science","algoliaApiEndpoint":"https://MQ57UUUQZ2-dsn.algolia.net","algoliaAppId":"MQ57UUUQZ2","algoliaSearchOnlyApiKey":"394474ced050e3911ae2249ecc774921","iosAppStoreUrl":"https://itunes.apple.com/app/medium-everyones-stories/id828256236?pt=698524&mt=8","iosAppLinkBaseUrl":"medium:","algoliaIndexPrefix":"medium_","androidPlayStoreUrl":"https://play.google.com/store/apps/details?id=com.medium.reader","googleClientId":"216296035834-k1k6qe060s2tp2a2jam4ljdcms00sttg.apps.googleusercontent.com","androidPackage":"com.medium.reader","androidPlayStoreMarketScheme":"market://details?id=com.medium.reader","googleAuthUri":"https://accounts.google.com/o/oauth2/auth","androidScheme":"medium","layoutData":{"useDynamicScripts":false,"googleAnalyticsTrackingCode":"UA-24232453-2","jsShivUrl":"https://cdn-static-1.medium.com/_/fp/js/shiv.RI2ePTZ5gFmMgLzG5bEVAA.js","useDynamicCss":false,"faviconUrl":"https://cdn-static-1.medium.com/_/fp/icons/favicon-rebrand-medium.3Y6xpZ-0FSdWDnPM3hSBIA.ico","faviconImageId":"1*8I-HPL0bfoIzGied-dzOvA.png","fontSets":[{"id":8,"url":"https://glyph.medium.com/css/e/sr/latin/e/ssr/latin/e/ssb/latin/m2.css"},{"id":11,"url":"https://glyph.medium.com/css/m2.css"},{"id":9,"url":"https://glyph.medium.com/css/mkt.css"}],"editorFaviconUrl":"https://cdn-static-1.medium.com/_/fp/icons/favicon-rebrand-medium-editor.3Y6xpZ-0FSdWDnPM3hSBIA.ico","glyphUrl":"https://glyph.medium.com"},"authBaseUrlRev":"moc.muidem//:sptth","isDnt":false,"stripePublishableKey":"pk_live_7FReX44VnNIInZwrIIx6ghjl","archiveUploadSizeMb":100,"paymentData":{"currencies":{"1":{"label":"US Dollar","external":"usd"}},"countries":{"1":{"label":"United States of America","external":"US"}},"accountTypes":{"1":{"label":"Individual","external":"individual"},"2":{"label":"Company","external":"company"}}},"previewConfig2":{"weightThreshold":1,"weightImageParagraph":0.05,"raiseImage":true,"enforceHeaderHierarchy":true,"isImageInsetRight":true},"isAmp":false,"iosScheme":"medium","isSwBoot":false,"lightstep":{"accessToken":"ce5be895bef60919541332990ac9fef2","carrier":"{\"ot-tracer-spanid\":\"00a375c8518525a8\",\"ot-tracer-traceid\":\"3e66db8f21a88dac\",\"ot-tracer-sampled\":\"true\"}","host":"collector-medium.lightstep.com"},"facebook":{"key":"542599432471018","namespace":"medium-com","scope":{"default":["public_profile","email"],"connect":["public_profile","email"],"login":["public_profile","email"],"share":["public_profile","email"]}},"editorsPicksTopicId":"3985d2a191c5","popularOnMediumTopicId":"9d34e48ecf94","memberContentTopicId":"13d7efd82fb2","audioContentTopicId":"3792abbd134","brandedSequenceId":"7d337ddf1941","isDoNotAuth":false,"buggle":{"url":"https://buggle.medium.com","videoUrl":"https://cdn-videos-1.medium.com","audioUrl":"https://cdn-audio-1.medium.com"},"referrerType":3,"isMeteredOut":false,"meterConfig":{"maxUnlockCount":3,"windowLength":"MONTHLY"},"partnerProgramEmail":"partnerprogram@medium.com","userResearchPrompts":[{"promptId":"li_post_page","type":0,"url":"www.calendly.com"},{"promptId":"li_home_page","type":1,"url":"mediumuserfeedback.typeform.com/to/GcFjEO"},{"promptId":"li_profile_page","type":2,"url":"www.calendly.com"}],"recaptchaKey":"6LdAokEUAAAAAC7seICd4vtC8chDb3jIXDQulyUJ","signinWallCustomDomainCollectionIds":["3a8144eabfe3","336d898217ee","61061eb0c96b","138adf9c44c","819cc2aaeee0"],"countryCode":"CN","bypassMeter":false,"branchKey":"key_live_ofxXr2qTrrU9NqURK8ZwEhknBxiI6KBm","paypal":{"clientMode":"production","oneYearGift":{"name":"Medium Membership (1 Year, Digital Gift Code)","description":"Unlimited access to the best and brightest stories on Medium. Gift codes can be redeemed at medium.com/redeem.","price":"50.00","currency":"USD","sku":"membership-gift-1-yr"}},"collectionConfig":{"mediumOwnedAndOperatedCollectionIds":["544c7006046e","bcc38c8f6edf","444d13b52878","8d6b8a439e32","92d2092dc598","1285ba81cada","cb8577c9149e","8ccfed20cbb2","ae2a65f35510"]},"currentDigestUser":{"userId":"3751a3493996","createdAt":1557477875037,"enableDigestThirty":false}}
// ]]></script><script charset="UTF-8" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/main-base.bundle.CwzueaBQKTFG7DV3x_5v9A.js" async=""></script><script>// <![CDATA[
window["obvInit"]({"value":{"id":"4ebd4711d0ec","versionId":"8a7e008ce521","creatorId":"b3c7769e3e2f","creator":{"userId":"b3c7769e3e2f","name":"Aaron (Ari) Bornstein","username":"aribornstein","createdAt":1528230120981,"imageId":"1*9Y0zWLwh1nYuBZMetDnC7w.jpeg","backgroundImageId":"","bio":"\x3cMicrosoft Open Source Engineer\x3e I am an AI enthusiast with a passion for engaging with new technologies, history, and computational medicine.","twitterScreenName":"","socialStats":{"userId":"b3c7769e3e2f","usersFollowedCount":30,"usersFollowedByCount":1061,"type":"SocialStats"},"social":{"userId":"3751a3493996","targetUserId":"b3c7769e3e2f","type":"Social"},"facebookAccountId":"","allowNotes":1,"mediumMemberAt":0,"isNsfw":false,"isWriterProgramEnrolled":true,"isQuarantined":false,"type":"User"},"homeCollection":{"id":"7f60cf5620c9","name":"Towards Data Science","slug":"towards-data-science","tags":["DATA SCIENCE","MACHINE LEARNING","ARTIFICIAL INTELLIGENCE","ANALYTICS","PROGRAMMING"],"creatorId":"895063a310f4","description":"Sharing concepts, ideas, and codes.","shortDescription":"Sharing concepts, ideas, and codes.","image":{"imageId":"1*F0LADxTtsKOgmPa-_7iUEQ.jpeg","filter":"","backgroundSize":"","originalWidth":1275,"originalHeight":1275,"strategy":"resample","height":0,"width":0},"metadata":{"followerCount":233425,"activeAt":1561508820290},"virtuals":{"permissions":{"canPublish":false,"canPublishAll":false,"canRepublish":false,"canRemove":false,"canManageAll":false,"canSubmit":false,"canEditPosts":false,"canAddWriters":false,"canViewStats":false,"canSendNewsletter":false,"canViewLockedPosts":false,"canViewCloaked":false,"canEditOwnPosts":false,"canBeAssignedAuthor":false,"canEnrollInHightower":false,"canLockPostsForMediumMembers":false,"canLockOwnPostsForMediumMembers":false},"isSubscribed":false,"isNewsletterSubscribed":false,"isEnrolledInHightower":false,"isEligibleForHightower":false,"mediumNewsletterId":"","isSubscribedToMediumNewsletter":false},"logo":{"imageId":"1*5EUO1kUYBthpOCPzRj_l2g.png","filter":"","backgroundSize":"","originalWidth":1010,"originalHeight":376,"strategy":"resample","height":0,"width":0},"twitterUsername":"TDataScience","facebookPageName":"towardsdatascience","collectionMastheadId":"8b6aceffde6","domain":"towardsdatascience.com","sections":[{"type":2,"collectionHeaderMetadata":{"title":"Towards Data Science","description":"Sharing concepts, ideas, and codes","backgroundImage":{},"logoImage":{},"alignment":2,"layout":5}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["7ffbd3d399a3","7182cf4b87ff"]}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":3,"postIds":["e755dd2f9ccf","80d2d512f155","7cdef669aeed"],"sectionHeader":"Featured "}},{"type":1,"postListMetadata":{"source":1,"layout":4,"number":6,"postIds":[],"sectionHeader":"Latest"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["66a6d3aebc97","6172d9c931a5"],"sectionHeader":"Our Letters"}},{"type":3,"promoMetadata":{"sectionHeader":"","promoId":"3245ee5b4331"}},{"type":1,"postListMetadata":{"source":2,"layout":4,"number":6,"postIds":[],"sectionHeader":"Trending"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["3bf37f75a345","3920888f831c"],"sectionHeader":"Our Readers’ Guide"}},{"type":1,"postListMetadata":{"source":4,"layout":4,"number":9,"postIds":[],"tagSlug":"Towards Data Science","sectionHeader":"Editors' Picks"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["96667b06af5","d691af11cc2f"],"sectionHeader":"Contribute"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["eea5e903499c","766cdd74d13e"]}},{"type":1,"postListMetadata":{"source":4,"layout":5,"number":3,"postIds":[],"tagSlug":"Towards Data Science","sectionHeader":"Last Chance To Read"}}],"tintColor":"#FF355876","lightText":true,"favicon":{"imageId":"1*F0LADxTtsKOgmPa-_7iUEQ.jpeg","filter":"","backgroundSize":"","originalWidth":1275,"originalHeight":1275,"strategy":"resample","height":0,"width":0},"colorPalette":{"defaultBackgroundSpectrum":{"colorPoints":[{"color":"#FF668AAA","point":0},{"color":"#FF61809D","point":0.1},{"color":"#FF5A7690","point":0.2},{"color":"#FF546C83","point":0.3},{"color":"#FF4D6275","point":0.4},{"color":"#FF455768","point":0.5},{"color":"#FF3D4C5A","point":0.6},{"color":"#FF34414C","point":0.7},{"color":"#FF2B353E","point":0.8},{"color":"#FF21282F","point":0.9},{"color":"#FF161B1F","point":1}],"backgroundColor":"#FFFFFFFF"},"tintBackgroundSpectrum":{"colorPoints":[{"color":"#FF355876","point":0},{"color":"#FF4D6C88","point":0.1},{"color":"#FF637F99","point":0.2},{"color":"#FF7791A8","point":0.3},{"color":"#FF8CA2B7","point":0.4},{"color":"#FF9FB3C6","point":0.5},{"color":"#FFB2C3D4","point":0.6},{"color":"#FFC5D2E1","point":0.7},{"color":"#FFD7E2EE","point":0.8},{"color":"#FFE9F1FA","point":0.9},{"color":"#FFFBFFFF","point":1}],"backgroundColor":"#FF355876"},"highlightSpectrum":{"colorPoints":[{"color":"#FFEDF4FC","point":0},{"color":"#FFE9F2FD","point":0.1},{"color":"#FFE6F1FD","point":0.2},{"color":"#FFE2EFFD","point":0.3},{"color":"#FFDFEEFD","point":0.4},{"color":"#FFDBECFE","point":0.5},{"color":"#FFD7EBFE","point":0.6},{"color":"#FFD4E9FE","point":0.7},{"color":"#FFD0E7FF","point":0.8},{"color":"#FFCCE6FF","point":0.9},{"color":"#FFC8E4FF","point":1}],"backgroundColor":"#FFFFFFFF"}},"navItems":[{"type":4,"title":"Data Science","url":"https://towardsdatascience.com/data-science/home","topicId":"cf416843aadc","source":"topicId"},{"type":4,"title":"Machine Learning","url":"https://towardsdatascience.com/machine-learning/home","topicId":"a5c9b2f1cb6b","source":"topicId"},{"type":4,"title":"Programming","url":"https://towardsdatascience.com/programming/home","topicId":"41533a1dc73c","source":"topicId"},{"type":4,"title":"Visualization","url":"https://towardsdatascience.com/data-visualization/home","topicId":"825e6cb8b9ce","source":"topicId"},{"type":4,"title":"AI","url":"https://towardsdatascience.com/artificial-intelligence/home","topicId":"7f029b17bf96","source":"topicId"},{"type":4,"title":"Journalism","url":"https://towardsdatascience.com/data-journalism/home","topicId":"27a6ac3980c6","source":"topicId"},{"type":4,"title":"Picks","url":"https://towardsdatascience.com/editors-picks/home","topicId":"e81f4fc5ee6b","source":"topicId"},{"type":3,"title":"Contribute","url":"https://towardsdatascience.com/contribute/home"}],"colorBehavior":2,"instantArticlesState":0,"acceleratedMobilePagesState":0,"googleAnalyticsId":"UA-19707169-24","ampLogo":{"imageId":"","filter":"","backgroundSize":"","originalWidth":0,"originalHeight":0,"strategy":"resample","height":0,"width":0},"header":{"title":"Towards Data Science","description":"Sharing concepts, ideas, and codes","backgroundImage":{},"logoImage":{},"alignment":2,"layout":5},"paidForDomainAt":1509037374118,"type":"Collection"},"homeCollectionId":"7f60cf5620c9","title":"Beyond Word Embeddings Part 2- Word Vectors & NLP Modeling from BoW to BERT","detectedLanguage":"en","latestVersion":"8a7e008ce521","latestPublishedVersion":"8a7e008ce521","hasUnpublishedEdits":false,"latestRev":2482,"createdAt":1538983314465,"updatedAt":1545049599327,"acceptedAt":0,"firstPublishedAt":1539794718913,"latestPublishedAt":1545049599327,"vote":false,"experimentalCss":"","displayAuthor":"","content":{"subtitle":"A primer in the neural nlp model archticture and word representation.","bodyModel":{"paragraphs":[{"name":"1d85","type":3,"text":"Beyond Word Embeddings Part 2","markups":[]},{"name":"1f51","type":1,"text":"Word Vectors and NLP Modeling from BoW to BERT","markups":[]},{"name":"c838","type":4,"text":"","markups":[],"layout":1,"metadata":{"id":"0*K8eg3bUVu4AG-4FB","isFeatured":true}},{"name":"081b","type":3,"text":"TL;DR","markups":[]},{"name":"5bd9","type":1,"text":"Since the advent of word2vec, neural word embeddings have become a go to method for encapsulating distributional semantics in text applications. This series will review the strengths and weaknesses of using pre-trained word embeddings and demonstrate how to incorporate more complex semantic representation schemes such as Semantic Role Labeling, Abstract Meaning Representation and Semantic Dependency Parsing into your applications.","markups":[{"type":3,"start":20,"end":28,"href":"https://en.wikipedia.org/wiki/Word2vec","title":"","rel":"noopener","anchorType":0}]},{"name":"746d","type":13,"text":"Introduction","markups":[]},{"name":"ecc8","type":1,"text":"The last post in this series reviewed some of the recent milestones in neural natural language processing. In this post we will review some of the advancements in text representation.","markups":[{"type":3,"start":4,"end":28,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f","title":"","rel":"","anchorType":0}]},{"name":"7db2","type":1,"text":"Computers are unable to understand the concepts of words. In order to process natural language a mechanism for representing text is required. The standard mechanism for text representation are word vectors where words or phrases from a given language vocabulary are mapped to vectors of real numbers.","markups":[{"type":3,"start":276,"end":283,"href":"https://en.wikipedia.org/wiki/Array_data_structure","title":"","rel":"","anchorType":0},{"type":3,"start":287,"end":299,"href":"https://en.wikipedia.org/wiki/Real_number","title":"","rel":"","anchorType":0},{"type":1,"start":193,"end":205},{"type":2,"start":211,"end":300}]},{"name":"9cfa","type":3,"text":"Traditional Word Vectors","markups":[{"type":1,"start":0,"end":24}]},{"name":"7ea6","type":1,"text":"Before diving directly into Word2Vec it’s worth while to do a brief overview of some of the traditional methods that pre-date neural embeddings.","markups":[]},{"name":"a41d","type":1,"text":"Bag of Words or BoW vector representations are the most common used traditional vector representation. Each word or n-gram is linked to a vector index and marked as 0 or 1 depending on whether it occurs in a given document.","markups":[{"type":1,"start":0,"end":12}]},{"name":"306e","type":4,"text":"An example of a one hot bag of words representation for documents with one word.","markups":[],"layout":1,"metadata":{"id":"1*dbx2P57arL8jrO-0QPrEKw.png","originalWidth":757,"originalHeight":163}},{"name":"7adf","type":1,"text":"BoW representations are often used in methods of document classification where the frequency of each word, bi-word or tri-word is a useful feature for training classifiers. One challenge with bag of word representations is that they don’t encode any information with regards to the meaning of a given word.","markups":[]},{"name":"76f0","type":1,"text":"In BoW word occurrences are evenly weighted independently of how frequently or what context they occur. However in most NLP tasks some words are more relevant than others.","markups":[]},{"name":"bcea","type":1,"text":"TF-IDF, short for term frequency–inverse document frequency, is a numerical statistic that is intended to reflect how important a word or n-gram is to a document in a collection or corpus. They provide some weighting to a given word based on the context it occurs.The tf–idf value increases proportionally to the number of times a word appears in a document and is offset by the number of documents in the corpus that contain the word, which helps to adjust for the fact that some words appear more frequently than others.","markups":[{"type":3,"start":18,"end":59,"href":"https://en.wikipedia.org/wiki/Tf%E2%80%93idf","title":"","rel":"","anchorType":0},{"type":3,"start":153,"end":161,"href":"https://en.wikipedia.org/wiki/Document","title":"Document","rel":"","anchorType":0},{"type":3,"start":181,"end":187,"href":"https://en.wikipedia.org/wiki/Text_corpus","title":"Text corpus","rel":"","anchorType":0},{"type":3,"start":291,"end":305,"href":"https://en.wikipedia.org/wiki/Proportionality_%28mathematics%29","title":"Proportionality (mathematics)","rel":"","anchorType":0},{"type":1,"start":0,"end":6},{"type":1,"start":18,"end":59}]},{"name":"70a6","type":4,"text":"https://skymind.ai/wiki/bagofwords-tf-idf","markups":[{"type":3,"start":0,"end":41,"href":"https://skymind.ai/wiki/bagofwords-tf-idf","title":"","rel":"nofollow","anchorType":0}],"layout":1,"metadata":{"id":"0*B67OTicNvuJaElW6.png","originalWidth":319,"originalHeight":200}},{"name":"9ed6","type":1,"text":"However even though tf-idf BoW representations provide weights to different words they are unable to capture the word meaning.","markups":[]},{"name":"3746","type":1,"text":"As the famous linguist J. R. Firth said in 1935, “The complete meaning of a word is always contextual, and no study of meaning apart from context can be taken seriously.”","markups":[{"type":2,"start":50,"end":169}]},{"name":"7b75","type":1,"text":"Distributional Embeddings enable word vectors to encapsulate contextual context. Each embedding vector is represented based on the mutual information it has with other words in a given corpus. Mutual information can be represented as a global co-occurrence frequency or restricted to a given window either sequentially or based on dependency edges.","markups":[{"type":1,"start":0,"end":26}]},{"name":"07c9","type":4,"text":"An example distributional embedding matrix each row encodes distributional context based on the count of the words it co-occurs with","markups":[],"layout":1,"metadata":{"id":"1*5oi3GdoCsk84JQGIVFmeAQ.png","originalWidth":262,"originalHeight":265}},{"name":"4a02","type":1,"text":"Distributional vectors predate neural methods for word embeddings and the techniques surrounding them are still relevant as they provide insight into better interpreting what neural embeddings learn. For more information one should read the work of Goldberg and Levy.","markups":[{"type":3,"start":249,"end":266,"href":"http://www.aclweb.org/anthology/Q15-1016","title":"","rel":"","anchorType":0}]},{"name":"16b1","type":3,"text":"Neural Embeddings","markups":[{"type":1,"start":0,"end":17}]},{"name":"3bca","type":1,"text":"Word2Vec","markups":[{"type":3,"start":0,"end":8,"href":"https://machinelearningmastery.com/develop-word-embeddings-python-gensim/","title":"","rel":"","anchorType":0},{"type":1,"start":0,"end":8}]},{"name":"3d70","type":1,"text":"Predictive models learn their vectors in order to improve their predictive ability of a loss such as the loss of predicting the vector for a target word from the vectors of the surrounding context words.","markups":[]},{"name":"21d0","type":1,"text":"Word2Vec is a predictive embedding model. There are two main Word2Vec architectures that are used to produce a distributed representation of words:","markups":[{"type":3,"start":111,"end":137,"href":"https://en.wikipedia.org/wiki/Distributed_representation","title":"Distributed representation","rel":"","anchorType":0}]},{"name":"b203","type":9,"text":"Continuous bag-of-words (CBOW) — The order of context words does not influence prediction (bag-of-words assumption). In the continuous skip-gram architecture, the model uses the current word to predict the surrounding window of context words.","markups":[{"type":3,"start":0,"end":23,"href":"https://en.wikipedia.org/wiki/Bag-of-words_model#CBOW","title":"","rel":"","anchorType":0},{"type":3,"start":91,"end":103,"href":"https://en.wikipedia.org/wiki/Bag-of-words","title":"Bag-of-words","rel":"noopener","anchorType":0}]},{"name":"b425","type":9,"text":"Continuous skip-gram weighs nearby context words more heavily than more distant context words. While order still is not captured each of the context vectors are weighed and compared independently vs CBOW which weighs against the average context.","markups":[{"type":3,"start":0,"end":20,"href":"https://en.wikipedia.org/wiki/N-gram#Skip-gram","title":"","rel":"noopener","anchorType":0}]},{"name":"6b59","type":4,"text":"CBOW and Skip-Gram Architectures","markups":[],"layout":1,"metadata":{"id":"0*TY9nYgPpwJloevhp.png","originalWidth":680,"originalHeight":286}},{"name":"849c","type":1,"text":"CBOW is faster while skip-gram is slower but does a better job for infrequent words.","markups":[]},{"name":"684d","type":1,"text":"GloVe","markups":[{"type":3,"start":0,"end":5,"href":"https://nlp.stanford.edu/projects/glove/","title":"","rel":"","anchorType":0},{"type":1,"start":0,"end":5}]},{"name":"1056","type":1,"text":"Both CBOW and Skip-Grams are “predictive” models, in that they only take local contexts into account. Word2Vec does not take advantage of global context. GloVe embeddings by contrast leverage the same intuition behind the co-occuring matrix used distributional embeddings, but uses neural methods to decompose the co-occurrence matrix into more expressive and dense word vectors. While GloVe vectors are faster to train, neither GloVe or Word2Vec has been shown to provide definitively better results rather they should both be evaluated for a given dataset.","markups":[{"type":3,"start":154,"end":159,"href":"https://nlp.stanford.edu/projects/glove/","title":"","rel":"","anchorType":0},{"type":1,"start":63,"end":100},{"type":1,"start":159,"end":160}]},{"name":"4c3d","type":1,"text":"FastText","markups":[{"type":3,"start":0,"end":8,"href":"https://fasttext.cc/","title":"","rel":"","anchorType":0},{"type":1,"start":0,"end":8}]},{"name":"fb95","type":1,"text":"FastText, builds on Word2Vec by learning vector representations for each word and the n-grams found within each word. The values of the representations are then averaged into one vector at each training step. While this adds a lot of additional computation to training it enables word embeddings to encode sub-word information. FastText vectors have been shown to be more accurate than Word2Vec vectors by a number of different measures","markups":[]},{"name":"d4f5","type":3,"text":"A 10,000 foot overview of Neural NLP Architectures","markups":[]},{"name":"4573","type":1,"text":"In addition to better word vector representation the advent of neural has led to advances in machine learning architectures that have enabled the advances listed in the previous post.","markups":[{"type":3,"start":169,"end":182,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f","title":"","rel":"","anchorType":0}]},{"name":"0f21","type":1,"text":"This section will highlight some of the key developments in neural architecture that enabled some of the NLP advances seen thus far. This not meant to be an exhaustive review of deep learning and machine learning NLP architecture, rather the goal is to demonstrate the changes that are driving NLP forward.","markups":[]},{"name":"b1de","type":13,"text":"Deep Feed Forward Networks","markups":[]},{"name":"eaf7","type":4,"text":"","markups":[],"layout":1,"metadata":{"id":"1*5CsWEdiDbInS2eZxgU3vKg.png"}},{"name":"5fc6","type":1,"text":"The advent of linear deep feed forward networks also known as multi layer perceptrons (MLP) in NLP introduced the potential for non linear modeling. This development helps with NLP because there are cases where the embedding space may be non linear. Take the following example of a documents whose embedding space is non linear meaning there is no way to linear divide the two document groups.","markups":[{"type":3,"start":62,"end":92,"href":"https://en.wikipedia.org/wiki/Multilayer_perceptron","title":"","rel":"","anchorType":0}]},{"name":"3313","type":4,"text":"It doesn’t matter how you fit a line there is no linear way to split the spam and ham documents","markups":[],"layout":1,"metadata":{"id":"0*R4cLnK1LdTUsxBuR.gif","originalWidth":1106,"originalHeight":330}},{"name":"5542","type":1,"text":"A non linear MLP network provides the ability to properly model such non linearities.","markups":[]},{"name":"e2bf","type":4,"text":"","markups":[],"layout":1,"metadata":{"id":"1*Q7JqxCpm-V8Pi69xtE12mA.png","originalWidth":884,"originalHeight":256}},{"name":"818f","type":1,"text":"This development by itself however did not bring about a significant revolution in NLP, since MLPs are unable to model word ordering. While MLPs open the door for marginal improvements in tasks such as language classification, where decisions can be made by modeling independent character frequencies, for more complex or ambiguous tasks standalone MLPs fall short.","markups":[]},{"name":"a812","type":13,"text":"1D CNNs","markups":[]},{"name":"a757","type":4,"text":"Kim, Y. (2014). Convolutional Neural Networks for Sentence Classification","markups":[{"type":2,"start":0,"end":73}],"layout":1,"metadata":{"id":"0*J3WBLXd8yFg8MAZp.png","originalWidth":1600,"originalHeight":645}},{"name":"0b8f","type":1,"text":"Prior to their application in NLP Convolutional Neural Networks (CNNs) provided groundbreaking results computer vision with the advent of AlexNet In NLP instead of convolving over pixels convulsion filters are applied and pooled sequentially over individual or groups of word vectors","markups":[{"type":3,"start":34,"end":71,"href":"https://en.wikipedia.org/wiki/Convolutional_neural_network","title":"","rel":"noopener","anchorType":0},{"type":3,"start":138,"end":145,"href":"https://en.wikipedia.org/wiki/AlexNet","title":"","rel":"noopener","anchorType":0}]},{"name":"aa21","type":1,"text":"In NLP CNNs are able to model local ordering by acting as n-gram feature extractors for embeddings. CNN models have contributed to state of the art results in classification and a variety of other NLP tasks.","markups":[{"type":1,"start":58,"end":98},{"type":2,"start":58,"end":98}]},{"name":"8c33","type":1,"text":"More recently the work of Jacovi and Golberg et al, has contributed to deeper understanding of what convolutional filters learn by demonstrating that filters are able to model rich semantic classes of n-grams by using different activation patterns, and that global max-pooling induces behavior which filters out less relevant n-grams from model decision process.","markups":[{"type":3,"start":26,"end":50,"href":"https://arxiv.org/abs/1809.08037","title":"","rel":"noopener","anchorType":0}]},{"name":"5817","type":1,"text":"A good primer on getting started with 1D CNNs can be found in the embedded link below.","markups":[]},{"name":"efb2","type":14,"text":"Deep Learning for Natural Language Processing — Part III\nIt’s been a month since I wrote the first part of this series. There, I shared the bit I know about word vector…medium.com","markups":[{"type":3,"start":0,"end":179,"href":"https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3","title":"https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3","rel":"","anchorType":0},{"type":1,"start":0,"end":56},{"type":2,"start":57,"end":169}],"mixtapeMetadata":{"mediaResourceId":"5b437c70f6fe532baaa11bd673585973","thumbnailImageId":"1*_tssg04hrYZlM6Ys4pUckw.png","href":"https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3"}},{"name":"7fe9","type":13,"text":"RNNs (LSTM/GRU)","markups":[]},{"name":"4d74","type":1,"text":"Building on the local ordering provide by CNNs Recurrent Neural Networks (RNNs) and their gated cell variants such as Long Short Term Memory Cells (LSTMs) and Gated Recurrent Units (GRUs) provide mechanisms for modeling sequential ordering and mid range dependencies in text such as the affect of a word in the beginning of a sentence on the end of a sentence.","markups":[]},{"name":"27d1","type":14,"text":"Illustrated Guide to LSTM’s and GRU’s: A step by step explanation\nHi and welcome to an Illustrated Guide to Long Short-Term Memory (LSTM) and Gated Recurrent Units (GRU). I’m Michael…towardsdatascience.com","markups":[{"type":3,"start":0,"end":205,"href":"https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21","title":"https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21","rel":"","anchorType":0},{"type":1,"start":0,"end":65},{"type":2,"start":66,"end":183}],"mixtapeMetadata":{"mediaResourceId":"3ce932bc502ce1fc9c6e50626db58deb","thumbnailImageId":"1*n-IgHZM5baBUjq0T7RYDBw.gif","href":"https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21"}},{"name":"b031","type":1,"text":"Additional variations of RNNs such as Bidirectional-RNNs which process text in both left to right and right to left and character level RNNs for enhancing underrepresented or out of vocabulary word embeddings led to many state of the art neural NLP breakthroughs.","markups":[{"type":3,"start":38,"end":56,"href":"https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks","title":"","rel":"","anchorType":0},{"type":3,"start":120,"end":141,"href":"https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html","title":"","rel":"","anchorType":0},{"type":3,"start":245,"end":262,"href":"http://karpathy.github.io/2015/05/21/rnn-effectiveness/","title":"","rel":"","anchorType":0}]},{"name":"0a60","type":4,"text":"An sample of some different RNN architectures and coupled with example use cases.","markups":[],"layout":1,"metadata":{"id":"0*MvzSPj3wF-nelK0j.gif","originalWidth":2667,"originalHeight":1500}},{"name":"3f3f","type":13,"text":"Attention and Copy Mechanisms","markups":[]},{"name":"dafe","type":1,"text":"While standard RNN architectures have led to incredible breakthroughs in NLP they suffer from a variety of challenges. While in theory they can capture long term dependencies they tend to struggle modeling longer sequences, this is still an open problem.","markups":[{"type":3,"start":56,"end":69,"href":"http://karpathy.github.io/2015/05/21/rnn-effectiveness/","title":"","rel":"","anchorType":0}]},{"name":"a65f","type":1,"text":"One cause for sub-optimal performance standard RNN encoder-decoder models for sequence to sequence tasks such as NER or translation is that they weight the impact each input vector evenly on each output vector when in reality specific words in the input sequence may carry more importance at different time steps.","markups":[{"type":3,"start":113,"end":116,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f","title":"","rel":"","anchorType":0}]},{"name":"deb7","type":1,"text":"Attention mechanisms provide a means of weighting the contextual impact of each input vector on each output prediction of the RNN. These mechanisms are responsible for much of the current or near current state of the art in Natural language processing.","markups":[{"type":3,"start":0,"end":20,"href":"https://skymind.ai/wiki/attention-mechanism-memory-network","title":"","rel":"","anchorType":0},{"type":1,"start":0,"end":20}]},{"name":"e771","type":4,"text":"An example of an attention mechanism applied to the task of neural translation in Microsoft Translator","markups":[],"layout":1,"metadata":{"id":"0*bVTfAB5K6aDSy1PG.gif","originalWidth":755,"originalHeight":731}},{"name":"7246","type":1,"text":"Additionally in Machine Reading Comprehension and Summarization systems RNNs often tend to generate results, that while on first glance look structurally correct are in reality hallucinated or incorrect. One mechanism that helps mitigate some of these issues is the Copy Mechanism.","markups":[{"type":3,"start":16,"end":45,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f","title":"","rel":"","anchorType":0},{"type":3,"start":50,"end":63,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f","title":"","rel":"","anchorType":0},{"type":3,"start":266,"end":280,"href":"http://www.abigailsee.com/2017/04/16/taming-rnns-for-better-summarization.html","title":"","rel":"","anchorType":0},{"type":1,"start":266,"end":281}]},{"name":"e30f","type":4,"text":"Copy Mechanism from Get To The Point: Summarization with Pointer-Generator Networks Abigail See, et all","markups":[],"layout":1,"metadata":{"id":"0*onHSVW8HkDma_EN2.png","originalWidth":1226,"originalHeight":800}},{"name":"bdba","type":1,"text":"The copy mechanism is an additional layer applied during decoding that decides whether it is better to generate the next word from the source sentence or from the general embedding vocabulary.","markups":[]},{"name":"7c14","type":13,"text":"Putting it all together with ELMo and BERT","markups":[]},{"name":"fdb6","type":1,"text":"ELMo is a model generates embeddings for a word based on the context it appears thus generating slightly different embeddings for each of its occurrence.","markups":[{"type":3,"start":0,"end":4,"href":"https://allennlp.org/elmo","title":"","rel":"","anchorType":0}]},{"name":"05a4","type":4,"text":"","markups":[],"layout":1,"metadata":{"id":"1*euk-3hzyi9nJvTdWFmfrqQ.png","originalWidth":580,"originalHeight":318}},{"name":"89ec","type":1,"text":"For example, the word “play” in the sentence above using standard word embeddings encodes multiple meanings such as the verb to play or in the case of the sentence a theatre production. In standard word embeddings such as Glove, Fast Text or Word2Vec each instance of the word play would have the same representation.","markups":[{"type":1,"start":23,"end":27},{"type":1,"start":125,"end":132},{"type":1,"start":277,"end":281},{"type":2,"start":23,"end":27},{"type":2,"start":125,"end":132},{"type":2,"start":277,"end":282}]},{"name":"1576","type":1,"text":"ELMo enables NLP models to better disambiguate between the correct sense of a given word. On in it’s release it enabled near instant state of the art results in many downstream tasks, including tasks such as co-reference were previously not as viable for practical usage.","markups":[]},{"name":"8151","type":4,"text":"","markups":[],"layout":1,"metadata":{"id":"1*BlrJnsOP_TxX1mgfT81I0Q.png","originalWidth":556,"originalHeight":442}},{"name":"4f97","type":1,"text":"ELMo also provides promising implications for preforming transfer learning on out of domain datasets. Some such as Sebastien Ruder have even hailed the coming ELMo as the ImageNet moment of NLP and while ELMo is a very promising development with practical real world applications, and has spawned recent related techniques such as BERT, that use attention transformers instead of bi-directonal RNNs to encode context, we will see in our upcoming post that there are still many obstacles in the world of Neural NLP.","markups":[{"type":3,"start":171,"end":193,"href":"http://ruder.io/nlp-imagenet/","title":"","rel":"","anchorType":0},{"type":3,"start":331,"end":335,"href":"https://arxiv.org/pdf/1810.04805.pdf","title":"","rel":"","anchorType":0}]},{"name":"9ed7","type":4,"text":"Comparsion of BERT and ELMo architectures from Devlin et. all","markups":[{"type":3,"start":14,"end":18,"href":"https://arxiv.org/pdf/1810.04805.pdf","title":"","rel":"","anchorType":0}],"layout":1,"metadata":{"id":"1*8WhXg3oXUC4s-m7F2ePLEA.png","originalWidth":617,"originalHeight":240}},{"name":"e042","type":3,"text":"Call To Action: Getting Started","markups":[]},{"name":"7773","type":1,"text":"Below are some resources to get started with the the different word embeddings above.","markups":[]},{"name":"3f9f","type":1,"text":"Documentation","markups":[{"type":1,"start":0,"end":13}]},{"name":"daef","type":9,"text":"Getting started with pyTorch and Docker on the Azure DLVM","markups":[{"type":3,"start":0,"end":57,"href":"https://docs.microsoft.com/en-us/learn/modules/interactive-deep-learning/?WT.mc_id=blog-medium-abornst","title":"","rel":"","anchorType":0}]},{"name":"bd6e","type":9,"text":"Character RNN classification with pyTorch","markups":[{"type":3,"start":0,"end":41,"href":"https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html","title":"","rel":"noopener","anchorType":0}]},{"name":"6da8","type":9,"text":"Fast Text Tutorial","markups":[{"type":3,"start":0,"end":18,"href":"https://fasttext.cc/docs/en/supervised-tutorial.html","title":"","rel":"noopener","anchorType":0}]},{"name":"2a4a","type":9,"text":"Keras NLP intro","markups":[{"type":3,"start":0,"end":15,"href":"https://nlpforhackers.io/keras-intro/","title":"","rel":"noopener","anchorType":0}]},{"name":"1345","type":1,"text":"Tools","markups":[{"type":1,"start":0,"end":5}]},{"name":"6106","type":9,"text":"Gensim","markups":[{"type":3,"start":0,"end":6,"href":"https://radimrehurek.com/gensim/","title":"","rel":"","anchorType":0}]},{"name":"af81","type":9,"text":"Word2Vec","markups":[{"type":3,"start":0,"end":8,"href":"https://radimrehurek.com/gensim/","title":"","rel":"","anchorType":0}]},{"name":"daaf","type":9,"text":"Sci-Kit Learn Bag of Words","markups":[{"type":3,"start":0,"end":26,"href":"http://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html","title":"","rel":"","anchorType":0}]},{"name":"7fe6","type":9,"text":"SpaCy Examples","markups":[{"type":3,"start":0,"end":14,"href":"https://spacy.io/usage/examples","title":"","rel":"","anchorType":0}]},{"name":"78d9","type":9,"text":"PyTorch Text","markups":[{"type":3,"start":0,"end":12,"href":"https://github.com/pytorch/text","title":"","rel":"","anchorType":0}]},{"name":"50dd","type":9,"text":"Allen NLP ELMo","markups":[{"type":3,"start":0,"end":14,"href":"https://github.com/allenai/allennlp/blob/master/tutorials/how_to/elmo.md","title":"","rel":"","anchorType":0}]},{"name":"9e9a","type":9,"text":"Hugging Face BERT PyTorch","markups":[{"type":3,"start":0,"end":25,"href":"https://github.com/huggingface/pytorch-pretrained-BERT","title":"","rel":"","anchorType":0}]},{"name":"47d5","type":9,"text":"Additional Resources","markups":[{"type":3,"start":0,"end":20,"href":"https://github.com/keon/awesome-nlp","title":"","rel":"","anchorType":0}]},{"name":"3fb9","type":1,"text":"Open Dataset","markups":[{"type":1,"start":0,"end":12}]},{"name":"ad2d","type":9,"text":"Dual Word Embeddings Trained on Bing Queries","markups":[{"type":3,"start":0,"end":44,"href":"https://msropendata.com/datasets/30a504b0-cff2-4d4a-864f-3bc9a66f9d7e","title":"","rel":"","anchorType":0}]},{"name":"c155","type":13,"text":"Next Post","markups":[{"type":3,"start":0,"end":9,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0","title":"","rel":"","anchorType":0}]},{"name":"36e1","type":1,"text":"Now that we have a solid understanding of some of the milestones in neural NLP, as well as the models and representations in the next post will review some of the pitfalls of current state of the art NLP systems.","markups":[{"type":3,"start":129,"end":211,"href":"https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0","title":"","rel":"","anchorType":0}]},{"name":"6b3c","type":1,"text":"If you have any questions, comments, or topics you would like me to discuss feel free to follow me on Twitter.","markups":[{"type":3,"start":102,"end":109,"href":"https://twitter.com/pythiccoder","title":"","rel":"noopener","anchorType":0}]},{"name":"9137","type":1,"text":"About the Author\nAaron (Ari) Bornstein is an avid AI enthusiast with a passion for history, engaging with new technologies and computational medicine. As an Open Source Engineer at Microsoft’s Cloud Developer Advocacy team, he collaborates with Israeli Hi-Tech Community, to solve real world problems with game changing technologies that are then documented, open sourced, and shared with the rest of the world.","markups":[{"type":2,"start":0,"end":17}]}],"sections":[{"name":"596c","startIndex":0},{"name":"2fc6","startIndex":5},{"name":"b50e","startIndex":8},{"name":"c2ba","startIndex":21},{"name":"76fb","startIndex":33}]},"postDisplay":{"coverless":true}},"virtuals":{"statusForCollection":"APPROVED","allowNotes":true,"previewImage":{"imageId":"0*K8eg3bUVu4AG-4FB","filter":"","backgroundSize":"","originalWidth":0,"originalHeight":0,"strategy":"resample","height":0,"width":0},"wordCount":2023,"imageCount":15,"readingTime":9.133962264150943,"subtitle":"A primer in the neural nlp model archticture and word representation.","userPostRelation":{"userId":"3751a3493996","postId":"4ebd4711d0ec","readAt":0,"readLaterAddedAt":0,"votedAt":0,"collaboratorAddedAt":0,"notesAddedAt":0,"subscribedAt":0,"lastReadSectionName":"","lastReadVersionId":"","lastReadAt":0,"lastReadParagraphName":"","lastReadPercentage":0,"viewedAt":0,"presentedCountInResponseManagement":0,"clapCount":0,"seriesUpdateNotifsOptedInAt":0,"queuedAt":0,"seriesFirstViewedAt":0,"presentedCountInStream":0,"seriesLastViewedAt":0,"audioProgressSec":0},"publishedInCount":1,"usersBySocialRecommends":[],"noIndex":false,"recommends":429,"socialRecommends":[],"isBookmarked":false,"tags":[{"slug":"machine-learning","name":"Machine Learning","postCount":74802,"metadata":{"postCount":74802,"coverImage":{"id":"0*8F8p5yS5x1OtGdCe.jpg","originalWidth":1680,"originalHeight":840,"isFeatured":true}},"type":"Tag"},{"slug":"nlp","name":"NLP","postCount":4765,"metadata":{"postCount":4765,"coverImage":{"id":"0*jzfLbSYA-VH5P7rC.png","originalWidth":904,"originalHeight":436,"isFeatured":true}},"type":"Tag"},{"slug":"ai","name":"AI","postCount":33422,"metadata":{"postCount":33422,"coverImage":{"id":"1*jL9fT-oAR6Ki3HOvXpwMLQ.png","originalWidth":1500,"originalHeight":843,"isFeatured":true}},"type":"Tag"},{"slug":"deep-learning","name":"Deep Learning","postCount":18437,"metadata":{"postCount":18437,"coverImage":{"id":"1*P7042_wRl6iE-STVPKzxNg.gif","originalWidth":1448,"originalHeight":788,"isFeatured":true}},"type":"Tag"},{"slug":"data-science","name":"Data Science","postCount":49618,"metadata":{"postCount":49618,"coverImage":{"id":"0*8F8p5yS5x1OtGdCe.jpg","originalWidth":1680,"originalHeight":840,"isFeatured":true}},"type":"Tag"}],"socialRecommendsCount":0,"responsesCreatedCount":4,"links":{"entries":[{"url":"https://twitter.com/pythiccoder","alts":[{"type":2,"url":"twitter://user?screen_name=pythiccoder"},{"type":3,"url":"twitter://user?screen_name=pythiccoder"}],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Word2vec","alts":[],"httpStatus":200},{"url":"https://docs.microsoft.com/en-us/learn/modules/interactive-deep-learning/?WT.mc_id=blog-medium-abornst","alts":[],"httpStatus":200},{"url":"http://www.abigailsee.com/2017/04/16/taming-rnns-for-better-summarization.html","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/AlexNet","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/N-gram#Skip-gram","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Text_corpus","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Document","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Bidirectional_recurrent_neural_networks","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Multilayer_perceptron","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Bag-of-words","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Proportionality_%28mathematics%29","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Tf%E2%80%93idf","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Bag-of-words_model#CBOW","alts":[],"httpStatus":200},{"url":"http://karpathy.github.io/2015/05/21/rnn-effectiveness/","alts":[],"httpStatus":200},{"url":"https://allennlp.org/elmo","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Convolutional_neural_network","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Array_data_structure","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Real_number","alts":[],"httpStatus":200},{"url":"http://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html","alts":[],"httpStatus":200},{"url":"https://en.wikipedia.org/wiki/Distributed_representation","alts":[],"httpStatus":200},{"url":"https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html","alts":[],"httpStatus":200},{"url":"https://fasttext.cc/","alts":[],"httpStatus":200},{"url":"https://fasttext.cc/docs/en/supervised-tutorial.html","alts":[],"httpStatus":200},{"url":"https://skymind.ai/wiki/attention-mechanism-memory-network","alts":[],"httpStatus":200},{"url":"https://skymind.ai/wiki/bagofwords-tf-idf","alts":[],"httpStatus":200},{"url":"https://machinelearningmastery.com/develop-word-embeddings-python-gensim/","alts":[],"httpStatus":200},{"url":"https://spacy.io/usage/examples","alts":[],"httpStatus":200},{"url":"https://medium.com/cityai/deep-learning-for-natural-language-processing-part-iii-96cfc6acfcc3","alts":[{"type":2,"url":"medium://p/96cfc6acfcc3"},{"type":3,"url":"medium://p/96cfc6acfcc3"}],"httpStatus":200},{"url":"https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21","alts":[{"type":2,"url":"medium://p/44e9eb85bf21"},{"type":3,"url":"medium://p/44e9eb85bf21"}],"httpStatus":200},{"url":"https://github.com/keon/awesome-nlp","alts":[],"httpStatus":200},{"url":"http://ruder.io/nlp-imagenet/","alts":[{"type":1,"url":"https://cdn.ampproject.org/c/ruder.io/nlp-imagenet/amp/"}],"httpStatus":200},{"url":"https://github.com/allenai/allennlp/blob/master/tutorials/how_to/elmo.md","alts":[],"httpStatus":200},{"url":"https://github.com/pytorch/text","alts":[],"httpStatus":200},{"url":"https://nlp.stanford.edu/projects/glove/","alts":[],"httpStatus":200},{"url":"https://msropendata.com/datasets/30a504b0-cff2-4d4a-864f-3bc9a66f9d7e","alts":[],"httpStatus":200},{"url":"https://radimrehurek.com/gensim/","alts":[],"httpStatus":200},{"url":"https://github.com/huggingface/pytorch-pretrained-BERT","alts":[],"httpStatus":200},{"url":"https://medium.com/@aribornstein/beyond-word-embeddings-part-1-an-overview-of-neural-nlp-milestones-82b97a47977f","alts":[{"type":2,"url":"medium://p/82b97a47977f"},{"type":3,"url":"medium://p/82b97a47977f"}],"httpStatus":200},{"url":"https://medium.com/@aribornstein/beyond-word-embeddings-part-3-four-common-flaws-in-state-of-the-art-neural-nlp-models-c1d35d3496d0","alts":[{"type":2,"url":"medium://p/c1d35d3496d0"},{"type":3,"url":"medium://p/c1d35d3496d0"}],"httpStatus":200},{"url":"http://www.aclweb.org/anthology/Q15-1016","alts":[],"httpStatus":200},{"url":"https://nlpforhackers.io/keras-intro/","alts":[{"type":1,"url":"https://cdn.ampproject.org/c/s/nlpforhackers.io/keras-intro/amp/"}],"httpStatus":200},{"url":"https://arxiv.org/abs/1809.08037","alts":[],"httpStatus":0},{"url":"https://arxiv.org/pdf/1810.04805.pdf","alts":[],"httpStatus":0}],"version":"0.3","generatedAt":1545049610499},"isLockedPreviewOnly":false,"metaDescription":"","totalClapCount":1937,"sectionCount":5,"readingList":0,"topics":[{"topicId":"1eca0103fff3","slug":"machine-learning","createdAt":1534449726145,"deletedAt":0,"image":{"id":"1*gFJS3amhZEg_z39D5EErVg@2x.png","originalWidth":2800,"originalHeight":1750},"name":"Machine Learning","description":"Teaching the learners.","relatedTopics":[],"visibility":1,"relatedTags":[],"relatedTopicIds":[],"type":"Topic"},{"topicId":"ae5d4995e225","slug":"data-science","createdAt":1493923906289,"deletedAt":0,"image":{"id":"1*NHWOEki_ncCX-xzbKtkEWw@2x.jpeg","originalWidth":5760,"originalHeight":3840},"name":"Data Science","description":"Query this.","relatedTopics":[],"visibility":1,"relatedTags":[],"relatedTopicIds":[],"type":"Topic"}]},"coverless":true,"slug":"beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert","translationSourcePostId":"","translationSourceCreatorId":"","isApprovedTranslation":false,"inResponseToPostId":"","inResponseToRemovedAt":0,"isTitleSynthesized":false,"allowResponses":true,"importedUrl":"","importedPublishedAt":0,"visibility":0,"uniqueSlug":"beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec","previewContent":{"bodyModel":{"paragraphs":[{"name":"previewImage","type":4,"text":"","layout":10,"metadata":{"id":"0*K8eg3bUVu4AG-4FB","isFeatured":true}},{"name":"previewTitle","type":3,"text":"Beyond Word Embeddings Part 2- Word Vectors & NLP Modeling from BoW to BERT","alignment":1},{"name":"previewSubtitle","type":13,"text":"A primer in the neural nlp…","alignment":1}],"sections":[{"startIndex":0}]},"isFullContent":false,"subtitle":"A primer in the neural nlp model archticture and word representation."},"license":0,"inResponseToMediaResourceId":"","canonicalUrl":"https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec","approvedHomeCollectionId":"7f60cf5620c9","approvedHomeCollection":{"id":"7f60cf5620c9","name":"Towards Data Science","slug":"towards-data-science","tags":["DATA SCIENCE","MACHINE LEARNING","ARTIFICIAL INTELLIGENCE","ANALYTICS","PROGRAMMING"],"creatorId":"895063a310f4","description":"Sharing concepts, ideas, and codes.","shortDescription":"Sharing concepts, ideas, and codes.","image":{"imageId":"1*F0LADxTtsKOgmPa-_7iUEQ.jpeg","filter":"","backgroundSize":"","originalWidth":1275,"originalHeight":1275,"strategy":"resample","height":0,"width":0},"metadata":{"followerCount":233425,"activeAt":1561508820290},"virtuals":{"permissions":{"canPublish":false,"canPublishAll":false,"canRepublish":false,"canRemove":false,"canManageAll":false,"canSubmit":false,"canEditPosts":false,"canAddWriters":false,"canViewStats":false,"canSendNewsletter":false,"canViewLockedPosts":false,"canViewCloaked":false,"canEditOwnPosts":false,"canBeAssignedAuthor":false,"canEnrollInHightower":false,"canLockPostsForMediumMembers":false,"canLockOwnPostsForMediumMembers":false},"isSubscribed":false,"isNewsletterSubscribed":false,"isEnrolledInHightower":false,"isEligibleForHightower":false,"mediumNewsletterId":"","isSubscribedToMediumNewsletter":false},"logo":{"imageId":"1*5EUO1kUYBthpOCPzRj_l2g.png","filter":"","backgroundSize":"","originalWidth":1010,"originalHeight":376,"strategy":"resample","height":0,"width":0},"twitterUsername":"TDataScience","facebookPageName":"towardsdatascience","collectionMastheadId":"8b6aceffde6","domain":"towardsdatascience.com","sections":[{"type":2,"collectionHeaderMetadata":{"title":"Towards Data Science","description":"Sharing concepts, ideas, and codes","backgroundImage":{},"logoImage":{},"alignment":2,"layout":5}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["7ffbd3d399a3","7182cf4b87ff"]}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":3,"postIds":["e755dd2f9ccf","80d2d512f155","7cdef669aeed"],"sectionHeader":"Featured "}},{"type":1,"postListMetadata":{"source":1,"layout":4,"number":6,"postIds":[],"sectionHeader":"Latest"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["66a6d3aebc97","6172d9c931a5"],"sectionHeader":"Our Letters"}},{"type":3,"promoMetadata":{"sectionHeader":"","promoId":"3245ee5b4331"}},{"type":1,"postListMetadata":{"source":2,"layout":4,"number":6,"postIds":[],"sectionHeader":"Trending"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["3bf37f75a345","3920888f831c"],"sectionHeader":"Our Readers’ Guide"}},{"type":1,"postListMetadata":{"source":4,"layout":4,"number":9,"postIds":[],"tagSlug":"Towards Data Science","sectionHeader":"Editors' Picks"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["96667b06af5","d691af11cc2f"],"sectionHeader":"Contribute"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["eea5e903499c","766cdd74d13e"]}},{"type":1,"postListMetadata":{"source":4,"layout":5,"number":3,"postIds":[],"tagSlug":"Towards Data Science","sectionHeader":"Last Chance To Read"}}],"tintColor":"#FF355876","lightText":true,"favicon":{"imageId":"1*F0LADxTtsKOgmPa-_7iUEQ.jpeg","filter":"","backgroundSize":"","originalWidth":1275,"originalHeight":1275,"strategy":"resample","height":0,"width":0},"colorPalette":{"defaultBackgroundSpectrum":{"colorPoints":[{"color":"#FF668AAA","point":0},{"color":"#FF61809D","point":0.1},{"color":"#FF5A7690","point":0.2},{"color":"#FF546C83","point":0.3},{"color":"#FF4D6275","point":0.4},{"color":"#FF455768","point":0.5},{"color":"#FF3D4C5A","point":0.6},{"color":"#FF34414C","point":0.7},{"color":"#FF2B353E","point":0.8},{"color":"#FF21282F","point":0.9},{"color":"#FF161B1F","point":1}],"backgroundColor":"#FFFFFFFF"},"tintBackgroundSpectrum":{"colorPoints":[{"color":"#FF355876","point":0},{"color":"#FF4D6C88","point":0.1},{"color":"#FF637F99","point":0.2},{"color":"#FF7791A8","point":0.3},{"color":"#FF8CA2B7","point":0.4},{"color":"#FF9FB3C6","point":0.5},{"color":"#FFB2C3D4","point":0.6},{"color":"#FFC5D2E1","point":0.7},{"color":"#FFD7E2EE","point":0.8},{"color":"#FFE9F1FA","point":0.9},{"color":"#FFFBFFFF","point":1}],"backgroundColor":"#FF355876"},"highlightSpectrum":{"colorPoints":[{"color":"#FFEDF4FC","point":0},{"color":"#FFE9F2FD","point":0.1},{"color":"#FFE6F1FD","point":0.2},{"color":"#FFE2EFFD","point":0.3},{"color":"#FFDFEEFD","point":0.4},{"color":"#FFDBECFE","point":0.5},{"color":"#FFD7EBFE","point":0.6},{"color":"#FFD4E9FE","point":0.7},{"color":"#FFD0E7FF","point":0.8},{"color":"#FFCCE6FF","point":0.9},{"color":"#FFC8E4FF","point":1}],"backgroundColor":"#FFFFFFFF"}},"navItems":[{"type":4,"title":"Data Science","url":"https://towardsdatascience.com/data-science/home","topicId":"cf416843aadc","source":"topicId"},{"type":4,"title":"Machine Learning","url":"https://towardsdatascience.com/machine-learning/home","topicId":"a5c9b2f1cb6b","source":"topicId"},{"type":4,"title":"Programming","url":"https://towardsdatascience.com/programming/home","topicId":"41533a1dc73c","source":"topicId"},{"type":4,"title":"Visualization","url":"https://towardsdatascience.com/data-visualization/home","topicId":"825e6cb8b9ce","source":"topicId"},{"type":4,"title":"AI","url":"https://towardsdatascience.com/artificial-intelligence/home","topicId":"7f029b17bf96","source":"topicId"},{"type":4,"title":"Journalism","url":"https://towardsdatascience.com/data-journalism/home","topicId":"27a6ac3980c6","source":"topicId"},{"type":4,"title":"Picks","url":"https://towardsdatascience.com/editors-picks/home","topicId":"e81f4fc5ee6b","source":"topicId"},{"type":3,"title":"Contribute","url":"https://towardsdatascience.com/contribute/home"}],"colorBehavior":2,"instantArticlesState":0,"acceleratedMobilePagesState":0,"googleAnalyticsId":"UA-19707169-24","ampLogo":{"imageId":"","filter":"","backgroundSize":"","originalWidth":0,"originalHeight":0,"strategy":"resample","height":0,"width":0},"header":{"title":"Towards Data Science","description":"Sharing concepts, ideas, and codes","backgroundImage":{},"logoImage":{},"alignment":2,"layout":5},"paidForDomainAt":1509037374118,"type":"Collection"},"newsletterId":"","webCanonicalUrl":"https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec","mediumUrl":"https://towardsdatascience.com/beyond-word-embeddings-part-2-word-vectors-nlp-modeling-from-bow-to-bert-4ebd4711d0ec","migrationId":"","notifyFollowers":true,"notifyTwitter":false,"notifyFacebook":false,"responseHiddenOnParentPostAt":0,"isSeries":false,"isSubscriptionLocked":false,"seriesLastAppendedAt":0,"audioVersionDurationSec":0,"sequenceId":"","isNsfw":false,"isEligibleForRevenue":false,"isBlockedFromHightower":false,"deletedAt":0,"lockedPostSource":0,"hightowerMinimumGuaranteeStartsAt":0,"hightowerMinimumGuaranteeEndsAt":0,"featureLockRequestAcceptedAt":0,"mongerRequestType":1,"layerCake":3,"socialTitle":"","socialDek":"","editorialPreviewTitle":"","editorialPreviewDek":"","curationEligibleAt":0,"primaryTopic":{"topicId":"1eca0103fff3","slug":"machine-learning","createdAt":1534449726145,"deletedAt":0,"image":{"id":"1*gFJS3amhZEg_z39D5EErVg@2x.png","originalWidth":2800,"originalHeight":1750},"name":"Machine Learning","description":"Teaching the learners.","relatedTopics":[],"visibility":1,"relatedTags":[],"relatedTopicIds":[],"type":"Topic"},"primaryTopicId":"1eca0103fff3","isProxyPost":false,"proxyPostFaviconUrl":"","proxyPostProviderName":"","proxyPostType":0,"type":"Post"},"mentionedUsers":[],"collaborators":[{"user":{"userId":"46b9b20c06bd","name":"Adi Polak","username":"adipolak","createdAt":1483864851965,"imageId":"2*aK-BSakOtnI7zrGyg99XuQ.png","backgroundImageId":"","bio":"Software & Stuff @Microsoft @Azure ❤️ Working with Kotlin, Scala, Linux, & Big Data Things 👩‍💻 Previously Sr. Software Engineer @Akamai 💅🌈","twitterScreenName":"adipolak","facebookAccountId":"10155990880548827","allowNotes":1,"mediumMemberAt":0,"isNsfw":false,"isWriterProgramEnrolled":true,"isQuarantined":false,"type":"User"},"state":"visible"}],"hideMeter":false,"collectionUserRelations":[],"mode":null,"references":{"User":{"b3c7769e3e2f":{"userId":"b3c7769e3e2f","name":"Aaron (Ari) Bornstein","username":"aribornstein","createdAt":1528230120981,"imageId":"1*9Y0zWLwh1nYuBZMetDnC7w.jpeg","backgroundImageId":"","bio":"\x3cMicrosoft Open Source Engineer\x3e I am an AI enthusiast with a passion for engaging with new technologies, history, and computational medicine.","twitterScreenName":"","socialStats":{"userId":"b3c7769e3e2f","usersFollowedCount":30,"usersFollowedByCount":1061,"type":"SocialStats"},"social":{"userId":"3751a3493996","targetUserId":"b3c7769e3e2f","type":"Social"},"facebookAccountId":"","allowNotes":1,"mediumMemberAt":0,"isNsfw":false,"isWriterProgramEnrolled":true,"isQuarantined":false,"type":"User"}},"Collection":{"7f60cf5620c9":{"id":"7f60cf5620c9","name":"Towards Data Science","slug":"towards-data-science","tags":["DATA SCIENCE","MACHINE LEARNING","ARTIFICIAL INTELLIGENCE","ANALYTICS","PROGRAMMING"],"creatorId":"895063a310f4","description":"Sharing concepts, ideas, and codes.","shortDescription":"Sharing concepts, ideas, and codes.","image":{"imageId":"1*F0LADxTtsKOgmPa-_7iUEQ.jpeg","filter":"","backgroundSize":"","originalWidth":1275,"originalHeight":1275,"strategy":"resample","height":0,"width":0},"metadata":{"followerCount":233425,"activeAt":1561508820290},"virtuals":{"permissions":{"canPublish":false,"canPublishAll":false,"canRepublish":false,"canRemove":false,"canManageAll":false,"canSubmit":false,"canEditPosts":false,"canAddWriters":false,"canViewStats":false,"canSendNewsletter":false,"canViewLockedPosts":false,"canViewCloaked":false,"canEditOwnPosts":false,"canBeAssignedAuthor":false,"canEnrollInHightower":false,"canLockPostsForMediumMembers":false,"canLockOwnPostsForMediumMembers":false},"isSubscribed":false,"isNewsletterSubscribed":false,"isEnrolledInHightower":false,"isEligibleForHightower":false,"mediumNewsletterId":"","isSubscribedToMediumNewsletter":false},"logo":{"imageId":"1*5EUO1kUYBthpOCPzRj_l2g.png","filter":"","backgroundSize":"","originalWidth":1010,"originalHeight":376,"strategy":"resample","height":0,"width":0},"twitterUsername":"TDataScience","facebookPageName":"towardsdatascience","collectionMastheadId":"8b6aceffde6","domain":"towardsdatascience.com","sections":[{"type":2,"collectionHeaderMetadata":{"title":"Towards Data Science","description":"Sharing concepts, ideas, and codes","backgroundImage":{},"logoImage":{},"alignment":2,"layout":5}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["7ffbd3d399a3","7182cf4b87ff"]}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":3,"postIds":["e755dd2f9ccf","80d2d512f155","7cdef669aeed"],"sectionHeader":"Featured "}},{"type":1,"postListMetadata":{"source":1,"layout":4,"number":6,"postIds":[],"sectionHeader":"Latest"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["66a6d3aebc97","6172d9c931a5"],"sectionHeader":"Our Letters"}},{"type":3,"promoMetadata":{"sectionHeader":"","promoId":"3245ee5b4331"}},{"type":1,"postListMetadata":{"source":2,"layout":4,"number":6,"postIds":[],"sectionHeader":"Trending"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["3bf37f75a345","3920888f831c"],"sectionHeader":"Our Readers’ Guide"}},{"type":1,"postListMetadata":{"source":4,"layout":4,"number":9,"postIds":[],"tagSlug":"Towards Data Science","sectionHeader":"Editors' Picks"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["96667b06af5","d691af11cc2f"],"sectionHeader":"Contribute"}},{"type":1,"postListMetadata":{"source":3,"layout":4,"number":2,"postIds":["eea5e903499c","766cdd74d13e"]}},{"type":1,"postListMetadata":{"source":4,"layout":5,"number":3,"postIds":[],"tagSlug":"Towards Data Science","sectionHeader":"Last Chance To Read"}}],"tintColor":"#FF355876","lightText":true,"favicon":{"imageId":"1*F0LADxTtsKOgmPa-_7iUEQ.jpeg","filter":"","backgroundSize":"","originalWidth":1275,"originalHeight":1275,"strategy":"resample","height":0,"width":0},"colorPalette":{"defaultBackgroundSpectrum":{"colorPoints":[{"color":"#FF668AAA","point":0},{"color":"#FF61809D","point":0.1},{"color":"#FF5A7690","point":0.2},{"color":"#FF546C83","point":0.3},{"color":"#FF4D6275","point":0.4},{"color":"#FF455768","point":0.5},{"color":"#FF3D4C5A","point":0.6},{"color":"#FF34414C","point":0.7},{"color":"#FF2B353E","point":0.8},{"color":"#FF21282F","point":0.9},{"color":"#FF161B1F","point":1}],"backgroundColor":"#FFFFFFFF"},"tintBackgroundSpectrum":{"colorPoints":[{"color":"#FF355876","point":0},{"color":"#FF4D6C88","point":0.1},{"color":"#FF637F99","point":0.2},{"color":"#FF7791A8","point":0.3},{"color":"#FF8CA2B7","point":0.4},{"color":"#FF9FB3C6","point":0.5},{"color":"#FFB2C3D4","point":0.6},{"color":"#FFC5D2E1","point":0.7},{"color":"#FFD7E2EE","point":0.8},{"color":"#FFE9F1FA","point":0.9},{"color":"#FFFBFFFF","point":1}],"backgroundColor":"#FF355876"},"highlightSpectrum":{"colorPoints":[{"color":"#FFEDF4FC","point":0},{"color":"#FFE9F2FD","point":0.1},{"color":"#FFE6F1FD","point":0.2},{"color":"#FFE2EFFD","point":0.3},{"color":"#FFDFEEFD","point":0.4},{"color":"#FFDBECFE","point":0.5},{"color":"#FFD7EBFE","point":0.6},{"color":"#FFD4E9FE","point":0.7},{"color":"#FFD0E7FF","point":0.8},{"color":"#FFCCE6FF","point":0.9},{"color":"#FFC8E4FF","point":1}],"backgroundColor":"#FFFFFFFF"}},"navItems":[{"type":4,"title":"Data Science","url":"https://towardsdatascience.com/data-science/home","topicId":"cf416843aadc","source":"topicId"},{"type":4,"title":"Machine Learning","url":"https://towardsdatascience.com/machine-learning/home","topicId":"a5c9b2f1cb6b","source":"topicId"},{"type":4,"title":"Programming","url":"https://towardsdatascience.com/programming/home","topicId":"41533a1dc73c","source":"topicId"},{"type":4,"title":"Visualization","url":"https://towardsdatascience.com/data-visualization/home","topicId":"825e6cb8b9ce","source":"topicId"},{"type":4,"title":"AI","url":"https://towardsdatascience.com/artificial-intelligence/home","topicId":"7f029b17bf96","source":"topicId"},{"type":4,"title":"Journalism","url":"https://towardsdatascience.com/data-journalism/home","topicId":"27a6ac3980c6","source":"topicId"},{"type":4,"title":"Picks","url":"https://towardsdatascience.com/editors-picks/home","topicId":"e81f4fc5ee6b","source":"topicId"},{"type":3,"title":"Contribute","url":"https://towardsdatascience.com/contribute/home"}],"colorBehavior":2,"instantArticlesState":0,"acceleratedMobilePagesState":0,"googleAnalyticsId":"UA-19707169-24","ampLogo":{"imageId":"","filter":"","backgroundSize":"","originalWidth":0,"originalHeight":0,"strategy":"resample","height":0,"width":0},"header":{"title":"Towards Data Science","description":"Sharing concepts, ideas, and codes","backgroundImage":{},"logoImage":{},"alignment":2,"layout":5},"paidForDomainAt":1509037374118,"type":"Collection"}},"Social":{"b3c7769e3e2f":{"userId":"3751a3493996","targetUserId":"b3c7769e3e2f","type":"Social"}},"SocialStats":{"b3c7769e3e2f":{"userId":"b3c7769e3e2f","usersFollowedCount":30,"usersFollowedByCount":1061,"type":"SocialStats"}}}})
// ]]></script><div class="surface-scrollOverlay"></div><script charset="UTF-8" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/main-common-async.bundle.mBB8x4lNtOgIEyw4wVO22w.js"></script><script charset="UTF-8" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/main-notes.bundle.m6hqQlEdhwrG8mHQcqEXww.js"></script><script>window.PARSELY = window.PARSELY || { autotrack: false }</script><script id="parsely-cfg" src="./Beyond Word Embeddings Part 2- Word Vectors &amp; NLP Modeling from BoW to BERT_files/p.js"></script><script type="text/javascript">(function(b,r,a,n,c,h,_,s,d,k){if(!b[n]||!b[n]._q){for(;s<_.length;)c(h,_[s++]);d=r.createElement(a);d.async=1;d.src="https://cdn.branch.io/branch-latest.min.js";k=r.getElementsByTagName(a)[0];k.parentNode.insertBefore(d,k);b[n]=h}})(window,document,"script","branch",function(b,r){b[r]=function(){b._q.push([r,arguments])}},{_q:[],_v:1},"addListener applyCode autoAppIndex banner closeBanner closeJourney creditHistory credits data deepview deepviewCta first getCode init link logout redeem referrals removeListener sendSMS setBranchViewData setIdentity track validateCode trackCommerceEvent logEvent".split(" "), 0); branch.init('key_live_ofxXr2qTrrU9NqURK8ZwEhknBxiI6KBm', {'no_journeys': true, 'disable_exit_animation': true, 'disable_entry_animation': true, 'tracking_disabled':  false }, function(err, data) {});</script></body></html>